cpgtools 2.0.5__py3-none-any.whl

cpgmodule/extend_bed.py

@@ -0,0 +1,147 @@
+import sys
+from bx.intervals import *
+import numpy as np
+from cpgmodule import ireader
+
+def getBasalDomains(bedfile, printit = False):
+	'''
+	Define gene's basal regulatory domain. 
+	bedfile: one gene one TSS (could use the canonical (longest) isoform, or merge all isoforms into a super transcript.
+	'''
+	basal_ranges = {}
+	
+	for l in ireader.reader(bedfile):
+		if l.startswith('#'):
+			continue
+		if l.startswith('track'):
+			continue
+		if l.startswith('browser'):
+			continue
+		f = l.split()
+		try:
+			chrom = f[0]
+			start = int(f[1])
+			end = int(f[2])
+			symbol = f[3]
+			gene_strand = f[5]
+			if start > end:
+				print ("'Start' cannot be larger than 'End'. Skip: " + l, file=sys.stderr)
+				continue
+			if gene_strand not in ['+','-']:
+				print ("Invalid strand. Skip: " + l, file=sys.stderr)
+				continue					
+		except:
+			print ("BED has at lesat 6 columns. Skip: " + l, file=sys.stderr)
+			continue
+
+		if chrom not in basal_ranges:
+			basal_ranges[chrom] = IntervalTree()
+		
+		basal_ranges[chrom].insert_interval( Interval(start, end, strand = gene_strand, value = symbol))
+		
+		if printit:
+			print('\t'.join([str(i) for i in (chrom, basal_st, basal_end, symbol, '0', strand)]), file = sys.stdout)
+	return basal_ranges
+
+def geteExtendedDomains(basal_ranges, bedfile, up_ext=2000, down_ext=2000, min_gene = 200, printit = False):
+	'''
+	Define gene's extended regulatory domain. 
+	bedfile:one gene one TSS (could use the canonical (longest) isoform, or merge all
+			isoforms into a super transcript.
+	up_ext:
+		Size of extension to upstream. Should be multiples of 100
+	down_ext:
+		Size of extension to downstream. Should be multiples of 100
+	min_gene:
+		minimum gene size (from TSS to TES). Should be multiples of 100
+	
+	'''	
+	return_ranges = []
+	
+	for l in ireader.reader(bedfile):
+		if l.startswith('#'):
+			continue
+		if l.startswith('track'):
+			continue
+		if l.startswith('browser'):
+			continue
+		f = l.split()
+		try:
+			chrom = f[0]
+			start = int(f[1])
+			end = int(f[2])
+			symbol = f[3]
+			strand = f[5]
+			
+			if start < 0:continue
+			if start > end:
+				print ("'Start' cannot be larger than 'End'. Skip: " + l, file=sys.stderr)
+				continue	
+			if (end - start ) < min_gene:
+				continue		
+			if strand not in ['+', '-']:
+				print ("Unknown strand. Skip: " + l, file=sys.stderr)
+				continue
+		except:
+			print ("BED has at lesat 6 columns. Skip: " + l, file=sys.stderr)
+
+		if strand == '+':
+			extension_st = start  - up_ext
+			extension_end = end + down_ext
+		elif strand == '-':
+			extension_st = start  - down_ext
+			extension_end = end + up_ext
+		if extension_st < 0: 
+			extension_st = 0
+		
+		#try to update extension_st
+		overlaps = basal_ranges[chrom].find(extension_st, start)
+		if len(overlaps) > 0:
+			for o in overlaps:
+				if o.end > extension_st:
+					extension_st = o.end
+			if extension_st > start:
+				extension_st = start
+		
+		if (start - extension_st) < min_gene:
+			continue
+		
+		#try to update extension_end
+		overlaps = basal_ranges[chrom].find(end, extension_end)
+		if len(overlaps) > 0:
+			for o in overlaps:
+				if o.start < extension_end:
+					extension_end = o.start
+			if extension_end < end:
+				extension_end = end
+		
+		if (extension_end - end) < min_gene:
+			continue
+		
+		return_ranges.append(([chrom, extension_st, start,symbol], [chrom, start, end,symbol], [chrom, end, extension_end,symbol], strand))
+		#return_ranges.append(([chrom, extension_st, start, symbol], [chrom, start, end, symbol], [chrom, end, extension_end,symbol], strand))
+		#return_ranges.append(([chrom, extension_st, start, strand], [chrom, start, end, strand], [chrom, end, extension_end, strand]))
+
+		if printit:
+			print('\t'.join([str(i) for i in (chrom, extension_st, extension_end, symbol, '0', strand,  start, end, '255,0,0', 1, extension_end - extension_st, 0)]), file = sys.stdout)
+		
+	return return_ranges
+
+if __name__=='__main__': 		
+	tmp = getBasalDomains(sys.argv[1], printit = False)	
+	b = geteExtendedDomains(basal_ranges = tmp, bedfile = sys.argv[1], printit=False)	
+	
+	
+	for a1,a2,a3,a4 in b:
+		if a4 == '+':
+			print ('\t'.join([str(i) for i in a1]) + '(UIR)\t' +str(int(a1[2]) - int(a1[1])) +  '\t+')
+			print ('\t'.join([str(i) for i in a2]) + '(Body)\t' +str(int(a2[2]) - int(a2[1])) +  '\t+')
+			print ('\t'.join([str(i) for i in a3]) + '(DIR)\t' +str(int(a3[2]) - int(a3[1])) +  '\t+')
+		if a4 == '-':
+			print ('\t'.join([str(i) for i in a1]) + '(DIR)\t' +str(int(a1[2]) - int(a1[1])) +  '\t-')
+			print ('\t'.join([str(i) for i in a2]) + '(Body)\t' +str(int(a2[2]) - int(a2[1])) +  '\t-')
+			print ('\t'.join([str(i) for i in a3]) + '(UIR)\t' +str(int(a3[2]) - int(a3[1])) +  '\t-')
+		
+		
+		
+		

cpgmodule/imotif.py

@@ -0,0 +1,348 @@
+#!/usr/bin/env python
+'''DNA/protein motif visualization and scan'''
+
+#import built-in modules
+import sys,os
+from collections import defaultdict
+from scipy import stats 
+import itertools
+
+#import third-party modules
+import numpy as np
+#changes to the paths
+
+#changing history to this module
+
+
+__author__ = "Liguo Wang"
+__copyright__ = ""
+__credits__ = []
+__license__ = "GPLv2"
+__version__ = "1.0.0"
+__maintainer__ = "Liguo Wang"
+__email__ = "Wang.Liguo@mayo.edu"
+__status__ = "Development" #Prototype or Production
+
+class PSSM (object):
+	'''
+	Description: provides functions to manipulate Position-Specific Scoring Matrix (PSSM) 
+	such as PFM, PPM, PWM matrix.
+	'''
+	
+	def __init__(self, sites, dna = True, name = None, rv = False):
+		'''
+		Initialize object.
+		Must be DNA or protein (dna = False) sequences.
+		dna = True: DNA sequence
+		dna = False: protein sequence
+		rv (reverse complementary): only applied to DNA sequence. 
+		
+		Each row contains a single sequence and each sequence has the same length.
+		Lowercase in sequence is automatically converted into uppercase.
+
+		Input example (test.sites):
+			GAGGTAAAC
+			TCCGTAAGT
+			CAGGTTGGA
+			ACAGTCAGT
+			TAGGTCATT
+			TAGGTACTG
+			ATGGTAACT
+			CAGGTATAC
+			TGTGTGAGT
+			AAGGTAAGT	
+		'''
+		if dna:
+			self.seq_type = 'DNA'
+		else:
+			self.seq_type = 'PROTEIN'
+		if name is None:
+			self.motif_name = 'Unknown'
+		else:
+			self.motif_name = name
+		if rv:
+			tab = string.maketrans('ACGT','TGCA')
+		self.seq_count = 0.0
+		self.data = defaultdict(dict)	#base_position (column of .sites): base_type : base_count
+		self.raw_data = defaultdict(list)	#base_position: list of ACGT in each column
+		self.seq_lengths = set()
+		self.DNA_bases = ['A','C','G','T']
+		self.protein_bases = ['A','R','N','D','C','Q','E','G','H','I','L','K','M','F','P','S','T','W','Y','V']
+		
+		for l in open(sites,'r'):
+			if l.startswith('#'):continue
+			if l.startswith('>'):continue
+			l = l.strip(' \r\n').upper()
+			
+			# check if all bases are valid symbols
+			skip = False
+			if self.seq_type == 'DNA':
+				for b in l:
+					if b not in  self.DNA_bases:
+						print("Uncognize DNA base: \"%s\" in %s. Skipped." % (b,l), file=sys.stderr)
+						skip = True
+						break
+			elif self.seq_type == 'PROTEIN':
+				for b in l:
+					if b not in  self.protein_bases:
+						print("Uncognize DNA base: \"%s\" in %s. Skipped." % (b,l), file=sys.stderr)
+						skip = True
+						break
+			if skip:
+				continue			
+			
+			self.seq_lengths.add(len(l))
+			self.seq_count += 1
+			
+			if rv:
+				l = l.translate(tab)[::-1]
+			
+			# read sites into dict(dict)
+			for i,v in enumerate(l):
+				self.raw_data[i].append(v)
+				if v not in self.data[i]:
+					self.data[i][v] = 1.0
+				else:
+					self.data[i][v] += 1.0
+		
+		# check if all sequences have the same length
+		if len(self.seq_lengths) != 1:
+			print("Sequence lengths are not equal!", file=sys.stderr)
+			sys.exit(1)
+		else:
+			self.motif_length =self.seq_lengths.pop()
+	
+	def motif_length(self):
+		'''
+		Return the motif length (nt)
+		'''
+		return self.motif_length	
+	
+	def toPFM(self,FOUT=sys.stdout):
+		'''
+		Convert motif sites data into position frequency matrix (PFM)	
+		'''	
+		pfm = []
+		if self.seq_type == 'DNA':
+			bases = self.DNA_bases
+		else:
+			bases = self.protein_bases		
+			
+		for i in range(self.motif_length):	# i is motif position starting from 0
+			tmp = []	# list of base_count for each column of sites file
+			for b in bases:
+				if b in self.data[i]:
+					tmp.append(self.data[i][b])
+				else:
+					tmp.append(0.0)
+			pfm.append(tmp)
+			
+		pfm = np.transpose(np.array(pfm))
+
+		#print("\n\n# PSSM matrix", file=FOUT)
+		print('Base\t' + '\t'.join([str(i+1) for i in range(self.motif_length)]), file=FOUT)
+		for i,name in enumerate(bases):
+			print(name + '\t' + '\t'.join([str(j) for j in pfm[i]]), file=FOUT)
+
+	def toJaspar(self,FOUT=sys.stdout):
+		'''
+		Convert motif sites data into Jaspar format (.pfm)
+		
+		Jaspar format example:
+			> Mycn
+			A [ 0 29 0 2 0 0 ]
+			C [31 0 30 1 3 0 ]
+			G [ 0 0 0 28 0 31]
+			T [ 0 2 1 0 28 0 ]		
+		'''	
+		pfm = []
+		if self.seq_type == 'DNA':
+			bases = self.DNA_bases
+		else:
+			bases = self.protein_bases		
+			
+		for i in range(self.motif_length):	# i is motif position starting from 0
+			tmp = []	# list of base_count for each column of sites file
+			for b in bases:
+				if b in self.data[i]:
+					tmp.append(self.data[i][b])
+				else:
+					tmp.append(0.0)
+			pfm.append(tmp)
+			
+		pfm = np.transpose(np.array(pfm))
+		print('> %s' % self.motif_name, file=FOUT)
+		for i,b in enumerate(bases):
+			print(b + ' [ ' + ' '.join([str(j) for j in pfm[i]]) + ']', file=FOUT)
+	
+	def toRawPSSM(self, FOUT=sys.stdout):
+		'''
+		Convert motif sites data into raw PSSM format (.pfm)
+		
+		raw PSSM format example:
+			>Mync
+			0 31 0 0
+			29 0 0 2
+			0 30 0 1
+			2 1 28 0
+			0 3 0 28
+			0 0 31 0		
+		'''	
+		pfm = []
+		if self.seq_type == 'DNA':
+			bases = self.DNA_bases
+		else:
+			bases = self.protein_bases		
+			
+		for i in range(self.motif_length):	# i is motif position starting from 0
+			tmp = []	# list of base_count for each column of sites file
+			for b in bases:
+				if b in self.data[i]:
+					tmp.append(self.data[i][b])
+				else:
+					tmp.append(0.0)
+			pfm.append(tmp)
+			
+		pfm = np.array(pfm)
+		print('>%s' % self.motif_name, file=FOUT)
+		for i in range(self.motif_length):
+			print(' '.join([str(j) for j in pfm[i]]), file=FOUT)
+
+	def toMEME(self,pseudocount=0.8, FOUT=sys.stdout):
+		'''
+		Convert motif sites data into meme's position-specific probability matrix
+		
+		MEME format example:
+			------------------------
+			Motif 2 position-specific probability matrix
+			------------------------
+			letter-probability matrix: alength= 4 w= 6 nsites= 31
+			0 31 0 0
+			29 0 0 2
+			0 30 0 1
+			2 1 28 0
+			0 3 0 28
+			0 0 31 0		
+		'''	
+		pfm = []
+		ppm = []
+
+		if self.seq_type == 'DNA':
+			bases = self.DNA_bases
+		else:
+			bases = self.protein_bases
+			
+		for i in range(self.motif_length):	# i is motif position starting from 0
+			tmp = []	# list of base_count for each column of sites file			
+			for b in bases:
+				if b in self.data[i]:
+					tmp.append(self.data[i][b])
+				else:
+					tmp.append(0.0)
+			pfm.append(tmp)
+			
+		pfm = np.transpose(np.array(pfm))
+		pfm = pfm + pseudocount/4.0
+		ppm = pfm/pfm.sum(axis=0)			
+		
+		ppm = np.transpose(ppm)
+		
+		print('-'*40, file=FOUT)
+		print(self.motif_name + ' position-specific probability matrix', file=FOUT)
+		print('-'*40, file=FOUT)
+		print('letter-probability matrix: alength= %d w= %d nsites= %d' % (len(bases), self.motif_length, self.seq_count), file=FOUT)
+		for i in ppm:
+			print(' ' + ' '.join([str(j) for j in i]), file=FOUT)
+		
+	def toPPM(self,pseudocount=0.8, FOUT=sys.stdout):
+		'''
+		Convert motif sites data into position probability matrix (PPM)
+		Default pseudocount of 0.8 is determined from this paper:
+		http://nar.oxfordjournals.org/content/37/3/939.full
+		'''	
+		pfm = []
+		ppm = []
+
+		if self.seq_type == 'DNA':
+			bases = self.DNA_bases
+		else:
+			bases = self.protein_bases
+			
+		for i in range(self.motif_length):	# i is motif position starting from 0
+			tmp = []	# list of base_count for each column of sites file			
+			for b in bases:
+				if b in self.data[i]:
+					tmp.append(self.data[i][b])
+				else:
+					tmp.append(0.0)
+			pfm.append(tmp)
+			
+		pfm = np.transpose(np.array(pfm))
+		pfm = pfm + pseudocount/4.0
+		ppm = pfm/pfm.sum(axis=0)			
+
+		#print("\n\n# PPM matrix", file=FOUT)
+		print('Base\t' + '\t'.join([str(i+1) for i in range(self.motif_length)]), file=FOUT)
+		for i,name in enumerate(bases):
+			print(name + '\t' + '\t'.join([str(j) for j in ppm[i]]), file=FOUT)
+
+	def toPWM(self,pseudocount=0.8, bg=None, FOUT=sys.stdout):
+		'''
+		Convert motif sites data into position weight matrix (PWM)
+		PWM is a matrix of log likelihood between sites and background.
+		
+		Default pseudocount of 0.8 is determined from this paper:
+		http://nar.oxfordjournals.org/content/37/3/939.full
+		
+		if bg is "None", universal background will be used:
+		 * DNA:
+		 	A = C = G = T = 0.25 (i.e. 1/4)
+		 * Protein
+		 	A = R = N = ... = V = 0.05 (i.e. 1/20)
+		 
+		 Otherwise, bg is a dictionary with base as key and the corresponding
+		 base frequency as value. eg
+		 
+		 bg = {'A':0.23, 'C':0.26,'G':0.29,'T':0.22}
+		
+		'''	
+		pwm = []
+		pfm = []
+		ppm = []
+		background = {}
+		
+		if self.seq_type == 'DNA':
+			bases = self.DNA_bases
+		else:
+			bases = self.protein_bases
+		
+		#determine background frequency
+		if bg is None:
+			for b in bases:
+				background[b] = 1.0/len(bases)
+						
+		for i in range(self.motif_length):	# i is motif position starting from 0
+			tmp = []	# list of base_count for each column of sites file
+			for b in bases:
+				if b in self.data[i]:
+					tmp.append(self.data[i][b])
+				else:
+					tmp.append(0.0)
+			pfm.append(tmp)
+			
+		pfm = np.transpose(np.array(pfm))
+		pfm = pfm + pseudocount/4.0
+		ppm = pfm/pfm.sum(axis=0)		
+		
+		for i,name in enumerate(bases):
+			tmp = [np.log(j/background[name]) for j in ppm[i]]
+			pwm.append(tmp)
+		
+		#print("\n\n# PWM matrix", file=FOUT)
+		print('Base\t' + '\t'.join([str(i+1) for i in range(self.motif_length)]), file=FOUT)
+		for i,name in enumerate(bases):
+			print(name + '\t' + '\t'.join([str(np.log(j/background[name])) for j in ppm[i]]), file=FOUT)
+
+
+	
+		

cpgmodule/ireader.py

@@ -0,0 +1,28 @@
+"""
+read compressed (.gz .bz) files
+"""
+#!/usr/bin/env python
+# encoding: utf-8
+
+import bz2
+import gzip
+from urllib.request import urlopen
+
+def nopen(f, mode="rb"):
+	if not isinstance(f, str):
+		return f
+	if f.startswith("|"):
+		p = Popen(f[1:], stdout=PIPE, stdin=PIPE, shell=True)
+		if mode[0] == "r": return p.stdout
+		return p
+	return {"r": sys.stdin, "w": sys.stdout}[mode[0]] if f == "-" \
+		else gzip.open(f, mode) if f.endswith((".gz", ".Z", ".z")) \
+		else bz2.BZ2File(f, mode) if f.endswith((".bz", ".bz2", ".bzip2")) \
+		else urlopen(f) if f.startswith(("http://", "https://","ftp://")) \
+		else open(f, mode)
+
+ 
+def reader(fname):
+	for l in nopen(fname):
+		yield l.decode('utf8').strip().replace("\r", "")
+	

cpgmodule/methylClock.py

@@ -0,0 +1,53 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+Created on Fri Nov 25 10:55:14 2022
+
+@author: Liguo Wang
+"""
+from cpgmodule import ireader
+#import sys,os
+
+class MethylSig():
+	"""
+	Pack DNA methylation signature file into object.
+
+	>>> from cpgmodule import methylClock
+	>>> a = methylClock.MethylAge(signature_file = 'coefBlup.tsv', signature_name = 'BLUP', signature_info="")
+	>>> a.name
+	'BLUP'
+	>>> a.Intercept
+	91.15396
+	>>> a.ncpg
+	319607
+	"""
+
+	def __init__(self, signature_file, signature_name, tissues = [], unit = '', signature_info = '', reference = '', pub_link = '', method = ''):
+		self.name = signature_name
+		self.info = signature_info
+		self.tissues = tissues
+		self.unit = unit
+		self.coef = {}
+		self.cpgs = []
+		self.ncpg = 0
+		self.Intercept = 0.0
+		self.ref = reference
+		self.pubmed = pub_link
+		self.method = method
+		for l in ireader.reader(signature_file):
+			if l.startswith('#'):
+				continue
+			f = l.split()
+			if l.startswith('Intercept'):
+				try:
+					self.Intercept = float(f[1])
+				except:
+					self.Intercept = 0.0
+			else:
+				self.cpgs.append(f[0])
+				self.ncpg  += 1
+				try:
+					self.coef[f[0]] = float(f[1])
+					#self.ncpg  += 1
+				except:
+					continue

cpgmodule/padjust.py

@@ -0,0 +1,58 @@
+#!/usr/bin/env python3
+
+# Copyright 2017 Francisco Pina Martins <f.pinamartins@gmail.com>
+# This file is part of structure_threader.
+# structure_threader is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# structure_threader is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with structure_threader. If not, see <http://www.gnu.org/licenses/>.
+
+# Taken from https://stackoverflow.com/a/21739593/3091595, ported to python 3
+# and improved readability.
+import numpy as np
+
+def multiple_testing_correction(pvalues, correction_type="FDR"):
+    """
+    Consistent with R - print
+    correct_pvalues_for_multiple_testing([0.0, 0.01, 0.029, 0.03, 0.031, 0.05,
+                                          0.069, 0.07, 0.071, 0.09, 0.1])
+    """
+    #from numpy import array, empty
+    pvalues = np.array(pvalues)
+    sample_size = pvalues.shape[0]
+    qvalues = np.empty(sample_size)
+    if correction_type == "Bonferroni":
+        # Bonferroni correction
+        qvalues = sample_size * pvalues
+    elif correction_type == "Bonferroni-Holm":
+        # Bonferroni-Holm correction
+        values = [(pvalue, i) for i, pvalue in enumerate(pvalues)]
+        values.sort()
+        for rank, vals in enumerate(values):
+            pvalue, i = vals
+            qvalues[i] = (sample_size-rank) * pvalue
+    elif correction_type == "FDR":
+        # Benjamini-Hochberg, AKA - FDR test
+        values = [(pvalue, i) for i, pvalue in enumerate(pvalues)]
+        values.sort()
+        values.reverse()
+        new_values = []
+        for i, vals in enumerate(values):
+            rank = sample_size - i
+            pvalue, index = vals
+            new_values.append((sample_size/rank) * pvalue)
+        for i in range(0, int(sample_size)-1):
+            if new_values[i] < new_values[i+1]:
+                new_values[i+1] = new_values[i]
+        for i, vals in enumerate(values):
+            pvalue, index = vals
+            qvalues[index] = new_values[i]
+    return qvalues