cpgtools 2.0.5__py3-none-any.whl

cpgtools-2.0.5.data/scripts/CpG_aggregation.py

@@ -0,0 +1,238 @@
+#!python
+
+"""
+Aggregate proportion values of a list of CpGs that located in give genomic regions
+(eg. CpG islands, promoters, exons, etc.).
+
+Outlier CpG will be removed if the probability of observing its proportion value is less
+than p-cutoff. For example, if alpha set to 0.05, and there are 10 CpGs (n = 10) located in a 
+particular genomic region, the p-cutoff of this genomic region is 0.005 (0.05/10). Supposing
+the total reads mapped to this region is 100, out of which 25 are methylated reads (i.e.,
+regional methylation level (beta) = 25/100 = 0.25) 
+
+The probability of observing CpG (3,10) is :
+	pbinom(q=3, size=10, prob=0.25) = 0.7759
+The probability of observing CpG (0,10) is :
+	pbinom(q=0, size=10, prob=0.25) = 0.05631
+The probability of observing CpG (16,21) is :
+	pbinom(q=16, size=21, prob=0.25, lower.tail=FALSE) = 1.19e-07 (outlier)
+
+
+**Example of input file**
+
+Chrom	Start	End	score
+chr10	100017748	100017749	3,10	
+chr10	100017769	100017770	0,10	
+chr10	100017853	100017854	16,21
+
+"""
+
+import sys,os
+import collections
+import subprocess
+import numpy as np
+from scipy.stats import binom
+
+from optparse import OptionParser
+from cpgmodule._version import __version__
+from cpgmodule import ireader
+from cpgmodule.utils import *
+from cpgmodule import BED
+import pandas as pd
+from bx.intervals import *
+
+__author__ = "Liguo Wang"
+__copyright__ = "Copyleft"
+__credits__ = []
+__license__ = "GPL"
+__maintainer__ = "Liguo Wang"
+__email__ = "wang.liguo@mayo.edu"
+__status__ = "Development"
+
+
+
+def buildIntervalTree(bed_file):
+	'''
+	Build interval tree from BED file. Input BED file must have at least 4 columns
+	'''        
+	ranges={}
+	printlog("reading "+ bed_file + '...')
+	for line in ireader.reader(bed_file):
+		if line.startswith("track"):continue
+		if line.startswith("#"):continue
+		if line.startswith('browser'):continue   
+		if line.startswith('Chrom'):continue	
+		fields = line.rstrip('\n ').split()
+		if len(fields) < 4:
+			continue
+		chrom = fields[0]
+		start = int(fields[1])
+		end = int(fields[2])
+		score = fields[3]
+		
+		if start < 0:
+			continue
+		if end < 0:
+			continue
+		if start > end:
+			continue
+		
+
+		if chrom not in ranges:
+			ranges[chrom] = Intersecter()
+			ranges[chrom].add_interval( Interval( start, end, value=score) )
+		else:
+			ranges[chrom].add_interval( Interval( start, end, value=score) )
+	return ranges
+
+def findIntervals(chrom, start, end, obj, a = 0.01, counts = True):
+	'''
+	obj is the IntervalTree object returned by "buildIntervalTree.
+	'''
+	hits = []	# list of proportion values
+	
+	if chrom not in obj:
+		return hits
+	else:
+		overlaps = obj[chrom].find(int(start), int(end))
+		for i in overlaps:
+			hits.append(i.value)
+	if len(hits) == 0:
+		if counts:
+			return(['N/A']*6)
+		else:
+			return(['N/A']*2)
+	
+	if counts:
+		methyl = []	#list of methylated read for each CpG
+		total = []	#list of total read for each CpG
+
+		for h in hits:
+			m, t = h.split(',')
+			methyl.append(int(m))
+			total.append(int(t))
+		ori_CpG_count = len(total)		#number of CpGs of a region
+		p_cut = a / ori_CpG_count
+		ori_methyl_sum = int(np.sum(methyl))	#total reads of a region
+		ori_total_sum = int(np.sum(total))	#total methylated reads of a region
+		
+		if ori_total_sum == 0:
+			return(['N/A']*6)
+		if ori_methyl_sum == 0 or ori_methyl_sum == ori_total_sum:
+			return([ori_CpG_count, ori_methyl_sum, ori_total_sum, ori_CpG_count, ori_methyl_sum, ori_total_sum])
+	
+	
+		region_beta = ori_methyl_sum/ori_total_sum	#average methylation level of *region*, equivalent to prob in binomial 
+
+		
+		new_methyl = []
+		new_total = []
+		for m, t in zip(methyl, total):
+			p = binom.cdf(k = m, n = t, p = region_beta)
+			#print (p, m, t)
+			if p < p_cut:
+				continue
+			if (1.0 - p) < p_cut:
+				continue
+			new_methyl.append(m)
+			new_total.append(t)
+		new_CpG_count = len(new_total)
+		new_methyl_sum = int(np.sum(new_methyl))
+		new_total_sum = int(np.sum(new_total))
+	
+		return([new_CpG_count, new_methyl_sum, new_total_sum, ori_CpG_count, ori_methyl_sum, ori_total_sum])
+	else:
+		CpG_count = len(hits)
+		avg_beta = np.mean([float(i) for i in hits])
+		return ([CpG_count, avg_beta])
+	
+	
+
+def main():
+	
+	usage="%prog [options]" + "\n"
+	parser = OptionParser(usage,version="%prog " + __version__)
+	parser.add_option("-i","--input_file",action="store",type="string",dest="input_file",help="Input CpG file in BED format. The first 3 columns contain \"Chrom\", \"Start\", and \"End\". The 4th column contains proportion values.")
+	parser.add_option("-a","--alpha",action="store",type='float', dest="alpha_cut", default=0.05, help="The chance of mistakingly assign a particular CpG as an outlier for each genomic region. Only applied to count data. default=%default" )
+	parser.add_option("-b","--bed",action="store",type="string",dest="bed_file",help="BED3+ file specifying the genomic regions.")
+	parser.add_option("-o","--output",action="store",type='string', dest="out_file",help="The prefix of the output file.")
+	parser.add_option("-t","--type",action="store",type='string', dest="data_type",help="Data type in the forth column. Must be one of \"count\" (eg 3,10) or \"beta\"(eg, 0.2)")
+	(options,args)=parser.parse_args()
+
+	if not (options.input_file):
+		print (__doc__)
+		parser.print_help()
+		sys.exit(101)
+	if not (options.data_type):
+		print (__doc__)
+		parser.print_help()
+		sys.exit(102)	
+	if not (options.out_file):
+		print (__doc__)
+		parser.print_help()
+		sys.exit(103)
+	
+	if options.alpha_cut < 0:
+		options.alpha_cut = 0.05
+	if options.alpha_cut > 1:
+		options.alpha_cut = 1	
+
+	tree = buildIntervalTree(options.input_file)
+	
+	OUT = open(options.out_file,'w')
+	if options.data_type == 'count':
+		print ("#chrom\tstart\tend\tN_CpG_filtered\tN_methyl_filtered\tN_total_filtered\tN_CpG_ori\tN_methy_ori\tN_total_ori", file=OUT)	
+		for line in ireader.reader(options.bed_file):
+			line = line.strip()
+			if line.startswith("track"):continue
+			if line.startswith("#"):continue
+			if line.startswith('browser'):continue   
+			if line.startswith('Chrom'):continue	
+
+		
+			f = line.split()
+			if len(f) < 3:
+				continue
+			try:
+				chrom = f[0]
+				start = int(f[1])
+				end = int(f[2])
+			except:
+				continue
+		
+			tmp = findIntervals(chrom, start, end, tree, a = options.alpha_cut, counts=True)
+			if len(tmp) == 0:
+				print ('\t'.join(f[0:3]) + '\t' + '\t'.join( ['N/A']*6), file=OUT)
+			else:
+				print ('\t'.join(f[0:3]) + '\t' + '\t'.join([str(i) for i in tmp]), file=OUT)
+	elif options.data_type == 'beta':
+		#print ("#chrom\tstart\tend\tN_CpG\tavg_beta", file=OUT)	
+		for line in ireader.reader(options.bed_file):
+			line = line.strip()
+			if line.startswith("track"):continue
+			if line.startswith("#"):continue
+			if line.startswith('browser'):continue   
+			if line.startswith('Chrom'):continue	
+
+			f = line.split()
+			if len(f) < 3:
+				continue
+			try:
+				chrom = f[0]
+				start = int(f[1])
+				end = int(f[2])
+			except:
+				continue
+		
+			tmp = findIntervals(chrom, start, end, tree, a = options.alpha_cut, counts=False)
+			if len(tmp) == 0:
+				print (line + '\t' + '\t'.join( ['N/A']*2), file=OUT)
+			else:
+				print (line + '\t' + '\t'.join([str(i) for i in tmp]), file=OUT)
+	else:
+		print ("-t (--type) must take the value of 'count' or 'beta'", file=sys.stderr)
+		sys.exit(102)
+	OUT.close()
+	
+if __name__=='__main__':
+	main()

cpgtools-2.0.5.data/scripts/CpG_anno_position.py

@@ -0,0 +1,156 @@
+#!python
+
+"""
+Description
+-----------
+This program annotates CpG by its position.
+
+Notes:
+- Input CpG and BED files must have at least three columns
+- If multiple regions from the annotation BED file are overlapped with the **same**
+  CpG site, their names will be concatenated together. 
+   
+"""
+
+import sys,os
+import collections
+import subprocess
+import numpy as np
+from os.path import basename
+from optparse import OptionParser
+from cpgmodule._version import __version__
+from cpgmodule import ireader
+from cpgmodule.utils import *
+from cpgmodule import BED
+import pandas as pd
+from bx.intervals import *
+
+__author__ = "Liguo Wang"
+__copyright__ = "Copyleft"
+__credits__ = []
+__license__ = "GPL"
+__maintainer__ = "Liguo Wang"
+__email__ = "wang.liguo@mayo.edu"
+__status__ = "Development"
+
+
+
+def buildIntervalTree(bed_file, window_size = 0):
+	'''
+	Build interval tree from annotation BED file.
+	window : add this to the middle of each region.
+	'''        
+	ranges={}
+	printlog("Build interval tree from annotation file: %s ..." % bed_file)
+	for line in ireader.reader(bed_file):
+		if line.startswith("track"):continue
+		if line.startswith("#"):continue
+		if line.startswith('browser'):continue   
+		fields = line.rstrip('\n ').split()
+		if len(fields) < 3:
+			continue
+		chrom = fields[0]
+		start = int(fields[1])
+		end = int(fields[2])
+		
+		if window_size > 0:
+			# window middle position
+			mid = int(start + (end - start)/2.0 )
+			if start < 0:
+				continue
+			if end < 0:
+				continue
+			if start > end:
+				continue
+			
+			# window start position
+			extension = int(window_size * 0.5)
+			w_start = mid - extension
+			if w_start < start:
+				w_start = start
+			
+			# window end position
+			w_end = mid + extension
+			if w_end > end:
+				w_end = end
+			
+		if len(fields) >= 4:
+			name = fields[3]
+		else:
+			name = fields[0] + ':' + fields[1] + '-' + fields[2]
+
+		if chrom not in ranges:
+			ranges[chrom] = Intersecter()
+			ranges[chrom].add_interval( Interval( start, end, value=name) )
+		else:
+			ranges[chrom].add_interval( Interval( start, end, value=name) )
+	return ranges
+
+def findIntervals(chrom, start, end, obj):
+	'''
+	obj is the IntervalTree object returned by "buildIntervalTree.
+	'''
+	hits = set()
+	if chrom not in obj:
+		return hits
+	else:
+		overlaps = obj[chrom].find(int(start), int(end))
+		for i in overlaps:
+			hits.add(i.value)
+	return sorted(hits)
+
+def main():
+	
+	usage="%prog [options]" + "\n"
+	parser = OptionParser(usage,version="%prog " + __version__)
+	parser.add_option("-i","--input_file",action="store",type="string",dest="input_file",help="Input CpG file in BED3+ format.")
+	parser.add_option("-a","--annotation",action="store",type="string",dest="anno_file",help="Input annotation file in BED3+ format.")
+	parser.add_option("-w","--window",action="store",type='int', dest="window_size", default=0, help="Size of window centering on the middle-point of each genomic region defined in the annotation BED file (i.e., window_size*0.5 will be extended to up- and down-stream from the middle point of each genomic region). if WINDOW_SIZE = 0, use the original region. default=%default" )
+	parser.add_option("-o","--output",action="store",type='string', dest="out_file",help="The prefix of the output file.")
+	parser.add_option("-l", "--header", action="store_true", dest="header", default=False, help="If True, the first row of input CpG file is header. default=%default")
+	(options,args)=parser.parse_args()
+
+	
+	if not (options.input_file):
+		print (__doc__)
+		#print ('You must specify input file(s)',file=sys.stderr)
+		parser.print_help()
+		sys.exit(101)
+	if not (options.out_file):
+		print (__doc__)
+		#print ('You must specify the output file',file=sys.stderr)
+		parser.print_help()
+		sys.exit(102)	
+	if not (options.anno_file):
+		print (__doc__)
+		#print ('You must specify the annotation file',file=sys.stderr)
+		parser.print_help()
+		sys.exit(103)
+	tree = buildIntervalTree(options.anno_file, window_size = options.window_size)
+	
+	OUT = open(options.out_file + '.anno.txt','w')
+	line_num = 0
+	printlog("Reading CpG file: %s ..." % options.input_file)
+	for line in ireader.reader(options.input_file):	 
+		fields = line.rstrip('\n ').split()
+		if len(fields) < 3:
+			continue
+		line_num += 1
+		f = line.split()
+		if (line_num == 1 and options.header):
+			print (line + '\t' +  basename(options.anno_file), file=OUT)
+		else:
+			chrom = f[0]
+			start = int(f[1])
+			end = int(f[2])
+			overlaps = findIntervals(chrom, start, end, tree)
+			if len(overlaps) > 0:
+				print (line + '\t' + ','.join(overlaps), file=OUT)
+			else:
+				print (line + '\tN/A', file=OUT)
+	
+	OUT.close()
+		
+if __name__=='__main__':
+	main()
+	

cpgtools-2.0.5.data/scripts/CpG_anno_probe.py

@@ -0,0 +1,112 @@
+#!python
+
+"""
+Description
+-----------
+This program adds comprehensive annotation information to each 450K/850K probe ID.
+"""
+
+import sys,os
+from optparse import OptionParser
+from cpgmodule import ireader
+from cpgmodule.utils import *
+from cpgmodule._version import __version__
+
+__author__ = "Liguo Wang"
+__copyright__ = "Copyleft"
+__credits__ = []
+__license__ = "GPL"
+__maintainer__ = "Liguo Wang"
+__email__ = "wang.liguo@mayo.edu"
+__status__ = "Development"
+
+
+def read_annotation(infile):
+	head = []
+	cpg_infor = {}
+	for l in ireader.reader(infile):
+		if l.startswith('probeID'):
+			head = l.split()[1:]
+		else:
+			f = l.split()
+			cgid = f[0]
+			anno = '\t'.join(f[1:])
+			cpg_infor[cgid] = anno
+	return (head, cpg_infor)
+	
+def main():
+	usage="%prog [options]" + "\n"
+	parser = OptionParser(usage,version="%prog " + __version__)
+	parser.add_option("-i","--input_file",action="store",type="string",dest="input_file",help="Input data file (Tab-separated) with a certain column containing 450K/850K array CpG IDs. This file can be regular text file or compressed file (.gz, .bz2).")
+	parser.add_option("-a","--annotation",action="store",type="string",dest="anno_file",help="Annotation file. This file can be regular text file or compressed file (.gz, .bz2).")
+	parser.add_option("-o","--output",action="store",type='string', dest="out_file",help="Prefix of the output file.")
+	parser.add_option("-p","--probe_column",action="store",type='int', dest="probe_col",default=0, help="The number of column that contains probe IDs. Note: the column index starts with 0. default=%default.")
+	parser.add_option("-l", "--header", action="store_true", dest="header", default=False, help="Input data file has a header row.")
+	(options,args)=parser.parse_args()
+	
+	if not (options.input_file):
+		print (__doc__)
+		parser.print_help()
+		sys.exit(101)
+
+	if not (options.anno_file):
+		print (__doc__)
+		parser.print_help()
+		sys.exit(102)
+				
+	if not (options.out_file):
+		print (__doc__)
+		parser.print_help()
+		sys.exit(103)	
+
+	if not os.path.isfile(options.input_file):
+		print ("Input data file \"%s\" does not exist\n" % options.input_file) 
+		sys.exit(104)
+	if not os.path.isfile(options.anno_file):
+		print ("Input annotation file \"%s\" does not exist\n" % options.input_file) 
+		sys.exit(105)
+	
+	printlog("Read annotation file \"%s\" ..." % (options.anno_file))	
+	(header, data)= read_annotation(options.anno_file)
+	
+	OUT = open(options.out_file + '.anno.txt','w')
+	printlog("Add annotation information to \"%s\" ..." % (options.input_file))	
+	line_num = 0
+	for l in ireader.reader(options.input_file):
+		line_num += 1
+		f = l.split()
+		if line_num == 1:
+			if options.header:
+				print (l + '\t' +  '\t'.join(header), file=OUT)
+			else:
+				print ('\t'.join(['NA']*len(f)) + '\t' +  '\t'.join(header), file=OUT)
+		else:
+			if options.probe_col >= len(f):
+				print ("Error: column ID must be smaller than %d!" % len(f), file=sys.stderr)
+				sys.exit(0)
+			cgid = f[options.probe_col]
+			try:
+				print (l + '\t'  + data[cgid],file=OUT)
+			except:
+				print (l + '\t' + '\t'.join(['NA']*len(header)), file=OUT)
+	OUT.close()		
+
+if __name__=='__main__':
+	main()
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+		

cpgtools-2.0.5.data/scripts/CpG_density_gene_centered.py

@@ -0,0 +1,107 @@
+#!python
+
+"""
+Description
+-----------
+This program calculates the CpG density (count) profile over gene body as well as its up-
+down-stream regions. It is useful to visualize how CpGs are distributed around genes.
+
+Specifically, the up-stream region, gene region (from TSS to TES) and down-stream region
+wil be equally divided into 100 bins, then CpG count was aggregated over a total of 300 bins
+from 5' to 3' (upstream bins, gene bins, downstrem bins).
+#==========================================================================================
+"""
+
+import sys
+import subprocess
+from optparse import OptionParser
+from cpgmodule import ireader
+from cpgmodule.utils import *
+from cpgmodule import BED
+from cpgmodule import extend_bed
+from cpgmodule._version import __version__
+
+__author__ = "Liguo Wang"
+__copyright__ = "Copyleft"
+__credits__ = []
+__license__ = "GPL"
+__maintainer__ = "Liguo Wang"
+__email__ = "wang.liguo@mayo.edu"
+__status__ = "Development"
+
+def main():
+
+	usage="%prog [options]" + "\n"
+	parser = OptionParser(usage,version="%prog " + __version__)
+	parser.add_option("-i","--input_file",action="store",type="string",dest="input_file",help="BED file specifying the C position. This BED file should have at least three columns (Chrom, ChromStart, ChromeEnd).  Note: the first base in a chromosome is numbered 0. This file can be a regular text file or compressed file (.gz, .bz2).")
+	parser.add_option("-r","--refgene",action="store",type="string",dest="gene_file",help="Reference gene model in standard BED6+ format.")
+	parser.add_option("-d","--downstream",action="store",type="int",dest="downstream_size",default=2000,help="Maximum extension size from TES (transcription end site) to down-stream to define the \"downstream intergenic region (DIR)\". Note: (1) The actual used DIR size can be smaller because the extending process could stop earlier if it reaches the boundary of another nearby gene. (2) If the actual used DIR size is smaller than cutoff defined by \"-c/--SizeCut\", the gene will be skipped.  default=%default (bp)")
+	parser.add_option("-u","--upstream",action="store",type="int",dest="upstream_size",default=2000,help="Maximum extension size from TSS (transcription start site) to up-stream to define the \"upstream intergenic region (UIR)\". Note: (1) The actual used UIR size can be smaller because the extending process could stop earlier if it reaches the boundary of another nearby gene. (2) If the actual used UIR size is smaller than cutoff defined by \"-c/--SizeCut\", the gene will be skipped. default=%default (bp)")
+	parser.add_option("-c","--SizeCut",action="store",type="int",dest="minimum_size",default=200,help="The minimum gene size. Gene size is defined as the genomic size between TSS and TES, including both exons and introns. default=%default (bp)")
+	parser.add_option("-o","--output",action="store",type='string', dest="out_file",help="The prefix of the output file.")
+	(options,args)=parser.parse_args()
+
+	print ()
+
+	if not (options.input_file):
+		print (__doc__)
+		parser.print_help()
+		sys.exit(101)
+
+	if not (options.gene_file):
+		print (__doc__)
+		parser.print_help()
+		sys.exit(102)
+
+	if not (options.out_file):
+		print (__doc__)
+		parser.print_help()
+		sys.exit(103)
+
+	FOUT = open(options.out_file + '.tsv','w')
+	ROUT = open(options.out_file + '.r','w')
+
+	#step1: read CpG file
+	printlog("Reading CpG file: \"%s\"" % (options.input_file))
+	cpg_ranges = read_CpG_bed(options.input_file)
+
+	#step2: read gene file
+	printlog("Reading reference gene model: \"%s\"" % (options.gene_file))
+	tmp1 = extend_bed.getBasalDomains(options.gene_file)
+	tmp2 = extend_bed.geteExtendedDomains(basal_ranges = tmp1, bedfile = options.gene_file, up_ext=options.upstream_size, down_ext=options.downstream_size, min_gene = options.minimum_size, printit = False)
+
+	printlog("Calculating CpG density ...")
+	#CpG density
+	(up_density, gene_density, down_density) = density_over_range(tmp2, cpg_ranges)
+
+	printlog("Wrting data to : \"%s\"" % (options.out_file + '.tsv'))
+	print ("Group\tPosition\tCpG_count", file=FOUT)
+	for ind in (sorted(up_density)):
+		print ("Upstream\t" + str(ind) + '\t' + str(up_density[ind]), file = FOUT)
+
+	for ind in (sorted(gene_density)):
+		print ("GeneBody\t" + str(ind) + '\t' + str(gene_density[ind]), file = FOUT)
+
+	for ind in (sorted(down_density)):
+		print ("Downstream\t" + str(ind) + '\t' + str(down_density[ind]), file = FOUT)
+	FOUT.close()
+
+	print ('pdf(file=\"%s\", width=10, height=5)' % (options.out_file + '.pdf'),file=ROUT)
+	print ("d <- read.table(file = '%s', header = T, sep='\\t')" % (options.out_file + '.tsv'), file = ROUT)
+	print ("x = 1:length(d$CpG_count)", file=ROUT)
+	print ("plot(x,d$CpG_count,type='l',col='red',lwd=1,xaxt='n',ylab='CpG count',xlab='')", file=ROUT)
+	print ("abline(v = c(102,203),col='blue', lty='dashed', lwd=0.5)", file=ROUT)
+	print ("text(x=c(0,102,203)+50, y=0.1, labels=c('Upstream', 'geneBody','Downstream'))", file=ROUT)
+	print ('dev.off()',file=ROUT)
+	ROUT.close()
+
+	printlog("Running R script to: '%s'" % (options.out_file + '.r'))
+	try:
+		subprocess.call("Rscript " + options.out_file + '.r', shell=True)
+	except:
+		print ("Cannot generate pdf file from " + options.out_file + '.r', file=sys.stderr)
+		pass
+
+if __name__=='__main__':
+	main()
+