wwpdb.utils.seqdb-v2 0.13__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- wwpdb/utils/seqdb_v2/BlastProcess.py +676 -0
- wwpdb/utils/seqdb_v2/FastaUtil.py +141 -0
- wwpdb/utils/seqdb_v2/FetchNcbiXml.py +140 -0
- wwpdb/utils/seqdb_v2/FetchUniProtEntry.py +96 -0
- wwpdb/utils/seqdb_v2/FetchUnpXml.py +323 -0
- wwpdb/utils/seqdb_v2/NcbiBlastService.py +176 -0
- wwpdb/utils/seqdb_v2/ReadNcbiBlastXml.py +233 -0
- wwpdb/utils/seqdb_v2/ReadNcbiSummary.py +128 -0
- wwpdb/utils/seqdb_v2/ReadNcbiXml.py +214 -0
- wwpdb/utils/seqdb_v2/ReadUnpBlastXml.py +266 -0
- wwpdb/utils/seqdb_v2/ReadUnpXml.py +722 -0
- wwpdb/utils/seqdb_v2/UnpBlastService.py +144 -0
- wwpdb/utils/seqdb_v2/__init__.py +5 -0
- wwpdb/utils/seqdb_v2/mmCIFUtil.py +64 -0
- wwpdb_utils_seqdb_v2-0.13.dist-info/METADATA +67 -0
- wwpdb_utils_seqdb_v2-0.13.dist-info/RECORD +17 -0
- wwpdb_utils_seqdb_v2-0.13.dist-info/WHEEL +4 -0
|
@@ -0,0 +1,141 @@
|
|
|
1
|
+
##
|
|
2
|
+
# File: FastaUtil.py
|
|
3
|
+
# Date: 30-Jan-2014 Jdw
|
|
4
|
+
#
|
|
5
|
+
import sys
|
|
6
|
+
from gzip import GzipFile
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
class FastaUtil:
|
|
10
|
+
"""Simple FASTA reader and writer with methods adjusted UniProt Fasta files -"""
|
|
11
|
+
|
|
12
|
+
def __init__(self, verbose=False, log=sys.stderr):
|
|
13
|
+
self.__verbose = verbose
|
|
14
|
+
self.__lfh = log
|
|
15
|
+
self.__debug = False
|
|
16
|
+
|
|
17
|
+
def loadFastaUniProt(self, fastaFilePath):
|
|
18
|
+
"""Read the the input FASTA file and return a dictionary sequence using the
|
|
19
|
+
first white-space delimited token on each comment line as dicitonary key.
|
|
20
|
+
|
|
21
|
+
A list of sequence id's is also returned.
|
|
22
|
+
"""
|
|
23
|
+
seqDict = {}
|
|
24
|
+
seqIdList = []
|
|
25
|
+
|
|
26
|
+
if fastaFilePath[-3:] == ".gz":
|
|
27
|
+
ifh = GzipFile(fastaFilePath)
|
|
28
|
+
else:
|
|
29
|
+
ifh = open(fastaFilePath)
|
|
30
|
+
|
|
31
|
+
for cmtLine, sequence in self.__read_record_fasta(ifh):
|
|
32
|
+
try:
|
|
33
|
+
org = ""
|
|
34
|
+
geneName = ""
|
|
35
|
+
dbName = ""
|
|
36
|
+
dbAccession = ""
|
|
37
|
+
dbIsoform = ""
|
|
38
|
+
seqId = ""
|
|
39
|
+
description = ""
|
|
40
|
+
ff = cmtLine[1:].split("|")
|
|
41
|
+
dbName = ff[0].upper()
|
|
42
|
+
seqId = ff[1]
|
|
43
|
+
tS = ff[2]
|
|
44
|
+
tt = seqId.split("-")
|
|
45
|
+
dbAccession = seqId
|
|
46
|
+
dbIsoform = ""
|
|
47
|
+
if len(tt) > 1:
|
|
48
|
+
dbAccession = tt[0]
|
|
49
|
+
dbIsoform = tt[1]
|
|
50
|
+
|
|
51
|
+
idx = tS.find("OS=")
|
|
52
|
+
if idx > 0:
|
|
53
|
+
description = tS[: idx - 1]
|
|
54
|
+
ff = tS[idx:].split("=")
|
|
55
|
+
if len(ff) == 3:
|
|
56
|
+
if ff[0] == "OS":
|
|
57
|
+
org = ff[1][:-2]
|
|
58
|
+
geneName = ff[2]
|
|
59
|
+
elif len(ff) > 1 and ff[0] == "OS":
|
|
60
|
+
org = ff[1]
|
|
61
|
+
except Exception as e: # noqa: BLE001
|
|
62
|
+
if self.__verbose:
|
|
63
|
+
self.__lfh.write(
|
|
64
|
+
"+FastaUtil.loadFastaUniProt() Read failing for file %s at %s err %s\n"
|
|
65
|
+
% (fastaFilePath, cmtLine, str(e))
|
|
66
|
+
)
|
|
67
|
+
ifh.close()
|
|
68
|
+
return seqIdList, seqDict
|
|
69
|
+
if seqId not in seqDict:
|
|
70
|
+
seqIdList.append(seqId)
|
|
71
|
+
seqDict[seqId] = {
|
|
72
|
+
"sequence": sequence.upper(),
|
|
73
|
+
"description": description,
|
|
74
|
+
"org": org,
|
|
75
|
+
"gene_name": geneName,
|
|
76
|
+
"db_accession": dbAccession,
|
|
77
|
+
"db_name": dbName,
|
|
78
|
+
"db_isoform": dbIsoform,
|
|
79
|
+
}
|
|
80
|
+
elif self.__debug:
|
|
81
|
+
self.__lfh.write(
|
|
82
|
+
"+FastaUtil.loadFastaUniProt() Duplicate sequence identifier %s\n%s\n" % (seqId, sequence)
|
|
83
|
+
)
|
|
84
|
+
|
|
85
|
+
if self.__verbose:
|
|
86
|
+
self.__lfh.write(
|
|
87
|
+
"+FastaUtil.loadFastaUniProt() Read %d sequences in %s\n" % (len(seqIdList), fastaFilePath)
|
|
88
|
+
)
|
|
89
|
+
|
|
90
|
+
ifh.close()
|
|
91
|
+
return seqIdList, seqDict
|
|
92
|
+
|
|
93
|
+
@staticmethod
|
|
94
|
+
def __read_record_fasta(ifh):
|
|
95
|
+
"""Returen the next FASTA record in the input file handle."""
|
|
96
|
+
comment, sequence = None, []
|
|
97
|
+
for line in ifh:
|
|
98
|
+
line = line.rstrip() # noqa: PLW2901
|
|
99
|
+
if line.startswith(">"):
|
|
100
|
+
if comment:
|
|
101
|
+
yield (comment, "".join(sequence))
|
|
102
|
+
comment, sequence = line, []
|
|
103
|
+
else:
|
|
104
|
+
sequence.append(line)
|
|
105
|
+
if comment:
|
|
106
|
+
yield (comment, "".join(sequence))
|
|
107
|
+
|
|
108
|
+
# def __writeFasta(self, ofh, seqDict, seqIdList, maxLineLength=70):
|
|
109
|
+
# """"""
|
|
110
|
+
# lCount = 0
|
|
111
|
+
# for seqId in seqIdList:
|
|
112
|
+
# sA = []
|
|
113
|
+
# sequence = seqDict[seqId]
|
|
114
|
+
# sA.append(">%s\n" % seqId)
|
|
115
|
+
# lCount += 1
|
|
116
|
+
# for i in range(0, len(sequence), maxLineLength):
|
|
117
|
+
# sA.append(sequence[i : i + maxLineLength] + "\n")
|
|
118
|
+
# result = "".join(sA)
|
|
119
|
+
# lCount += 1
|
|
120
|
+
# ofh.write(result)
|
|
121
|
+
# self.__lfh.write("+FastaUtil.__writeFasta() Formatted %d sequences in %d text lines\n" % (len(seqIdList), lCount))
|
|
122
|
+
|
|
123
|
+
# def __formatFasta(self, seqDict, seqIdList, maxLineLength=70):
|
|
124
|
+
# """"""
|
|
125
|
+
# sA = []
|
|
126
|
+
# lCount = 0
|
|
127
|
+
# for seqId in seqIdList:
|
|
128
|
+
# sequence = seqDict[seqId]
|
|
129
|
+
# sA.append(">%s\n" % seqId)
|
|
130
|
+
# lCount += 1
|
|
131
|
+
# for i in range(0, len(sequence), maxLineLength):
|
|
132
|
+
# sA.append(sequence[i : i + maxLineLength] + "\n")
|
|
133
|
+
# result = "".join(sA)
|
|
134
|
+
# lCount += 1
|
|
135
|
+
# self.__lfh.write("+FastaUtil.__formatFasta() Formatted %d sequences in %d text lines\n" % (len(seqIdList), lCount))
|
|
136
|
+
# return result
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
if __name__ == "__main__": # pragma: no cover
|
|
140
|
+
fa = FastaUtil(verbose=True, log=sys.stderr)
|
|
141
|
+
sList, sDict = fa.loadFastaUniProt("uniprot_sprot_varsplic.fasta")
|
|
@@ -0,0 +1,140 @@
|
|
|
1
|
+
"""
|
|
2
|
+
File: FetchNcbiXml.py
|
|
3
|
+
Author: Zukang Feng
|
|
4
|
+
Update: 24-May-2010
|
|
5
|
+
Version: 001 Initial version
|
|
6
|
+
|
|
7
|
+
Updates:
|
|
8
|
+
|
|
9
|
+
25-Feb-2013 jdw create a class version to return full XML entries -
|
|
10
|
+
20-Mar-2013 jdw Fix error in import path
|
|
11
|
+
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
__author__ = "Zukang Feng"
|
|
15
|
+
__email__ = "zfeng@rcsb.rutgers.edu"
|
|
16
|
+
__version__ = "V0.001"
|
|
17
|
+
|
|
18
|
+
import getopt
|
|
19
|
+
import sys
|
|
20
|
+
|
|
21
|
+
import requests
|
|
22
|
+
import urllib3
|
|
23
|
+
|
|
24
|
+
from wwpdb.utils.seqdb_v2.ReadNcbiSummary import ReadNcbiSummaryString
|
|
25
|
+
from wwpdb.utils.seqdb_v2.ReadNcbiXml import ReadNcbiXmlString
|
|
26
|
+
|
|
27
|
+
# To disable warning about not checking ssl certificates. Still needed?
|
|
28
|
+
urllib3.disable_warnings()
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
class FetchNcbiXml:
|
|
32
|
+
"""Using esummary.fcgi utility to get entry summary xml file from NCBI site and
|
|
33
|
+
using ReadNcbiSummary class to parse the result
|
|
34
|
+
"""
|
|
35
|
+
|
|
36
|
+
def __init__(self, qid, database, apikey=None):
|
|
37
|
+
self._id = qid
|
|
38
|
+
self._database = database
|
|
39
|
+
self._apikey = apikey
|
|
40
|
+
self._baseUrl = "https://eutils.ncbi.nlm.nih.gov/entrez/eutils/esummary.fcgi"
|
|
41
|
+
self._data = self._RequestNcbiXml()
|
|
42
|
+
|
|
43
|
+
def _RequestNcbiXml(self):
|
|
44
|
+
"""Request summary xml from NCBI site"""
|
|
45
|
+
params = {}
|
|
46
|
+
params["db"] = self._database
|
|
47
|
+
params["id"] = self._id
|
|
48
|
+
params["retmode"] = "xml"
|
|
49
|
+
if self._apikey:
|
|
50
|
+
params["api_key"] = self._apikey
|
|
51
|
+
try:
|
|
52
|
+
reqH = requests.get(self._baseUrl, params=params, verify=False, timeout=30) # noqa: S501
|
|
53
|
+
reqH.raise_for_status()
|
|
54
|
+
data = reqH.text
|
|
55
|
+
return data
|
|
56
|
+
except: # noqa: E722 pylint: disable=bare-except
|
|
57
|
+
return ""
|
|
58
|
+
|
|
59
|
+
def WriteNcbiXml(self, filename):
|
|
60
|
+
try:
|
|
61
|
+
file = open(filename, "w")
|
|
62
|
+
file.write(self._data)
|
|
63
|
+
file.close()
|
|
64
|
+
return True
|
|
65
|
+
except: # noqa: E722 pylint: disable=bare-except
|
|
66
|
+
return False
|
|
67
|
+
|
|
68
|
+
def ParseNcbiXmlData(self):
|
|
69
|
+
"""Using ReadNcbiSummary class to parse the result"""
|
|
70
|
+
readxml = ReadNcbiSummaryString(self._data)
|
|
71
|
+
return readxml.GetResult()
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
class FetchFullNcbiXml:
|
|
75
|
+
"""Using efetch.fcgi utility to get and parse the full entry xml file from NCBI site."""
|
|
76
|
+
|
|
77
|
+
def __init__(self, qid, database, apikey=None):
|
|
78
|
+
self._id = qid
|
|
79
|
+
self._database = database
|
|
80
|
+
self._apikey = apikey
|
|
81
|
+
self._baseUrl = "https://eutils.ncbi.nlm.nih.gov/entrez/eutils/efetch.fcgi"
|
|
82
|
+
self._data = self._RequestNcbiXml()
|
|
83
|
+
|
|
84
|
+
def _RequestNcbiXml(self):
|
|
85
|
+
"""Request summary xml from NCBI site"""
|
|
86
|
+
params = {}
|
|
87
|
+
params["db"] = self._database
|
|
88
|
+
params["id"] = self._id
|
|
89
|
+
params["retmode"] = "xml"
|
|
90
|
+
if self._apikey:
|
|
91
|
+
params["api_key"] = self._apikey
|
|
92
|
+
try:
|
|
93
|
+
reqH = requests.get(self._baseUrl, params=params, verify=False, timeout=30) # noqa: S501
|
|
94
|
+
reqH.raise_for_status()
|
|
95
|
+
data = reqH.text
|
|
96
|
+
return data
|
|
97
|
+
except: # noqa: E722 pylint: disable=bare-except
|
|
98
|
+
return ""
|
|
99
|
+
|
|
100
|
+
def WriteNcbiXml(self, filename):
|
|
101
|
+
file = open(filename, "w")
|
|
102
|
+
file.write(self._data)
|
|
103
|
+
file.close()
|
|
104
|
+
|
|
105
|
+
def ParseNcbiXmlData(self):
|
|
106
|
+
"""Parse xml result"""
|
|
107
|
+
readxml = ReadNcbiXmlString(self._data)
|
|
108
|
+
return readxml.GetResult()
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
def main(argv): # pragma: no cover
|
|
112
|
+
opts, _args = getopt.getopt(argv, "i:d:a:", ["id=", "db=", "apikey="])
|
|
113
|
+
|
|
114
|
+
tid = None
|
|
115
|
+
db = None
|
|
116
|
+
apikey = None
|
|
117
|
+
for opt, arg in opts:
|
|
118
|
+
if opt in ("-i", "--id"):
|
|
119
|
+
tid = arg
|
|
120
|
+
elif opt in ("-d", "--db"):
|
|
121
|
+
db = arg
|
|
122
|
+
elif opt in ("-a", "--apikey"):
|
|
123
|
+
apikey = arg
|
|
124
|
+
|
|
125
|
+
if tid and db:
|
|
126
|
+
fetchobj = FetchNcbiXml(tid, db, apikey)
|
|
127
|
+
# fetchobj = FetchFullNcbiXml(tid, db, apikey)
|
|
128
|
+
fetchobj.WriteNcbiXml(tid + ".xml")
|
|
129
|
+
pdict = fetchobj.ParseNcbiXmlData()
|
|
130
|
+
for k, v in pdict.items():
|
|
131
|
+
print("%s=%s" % (k, v)) # noqa: T201
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
if __name__ == "__main__": # pragma: no cover
|
|
135
|
+
try:
|
|
136
|
+
main(sys.argv[1:])
|
|
137
|
+
sys.exit(0)
|
|
138
|
+
except Exception as exc: # noqa: BLE001
|
|
139
|
+
print(exc) # noqa: T201
|
|
140
|
+
sys.exit(1)
|
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
##
|
|
2
|
+
# File: FetchUniprotEntry.py
|
|
3
|
+
# Author: John Westbrook
|
|
4
|
+
# Update: 29-July-2014
|
|
5
|
+
# Version: 001 Initial version
|
|
6
|
+
#
|
|
7
|
+
# Update:
|
|
8
|
+
# 17-Sep-2014 jdw add attribute to reflect isoform sequence replaced from FASTA resource file
|
|
9
|
+
#
|
|
10
|
+
##
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
import os
|
|
14
|
+
import sys
|
|
15
|
+
import traceback
|
|
16
|
+
|
|
17
|
+
from wwpdb.utils.config.ConfigInfoApp import ConfigInfoAppCommon
|
|
18
|
+
from wwpdb.utils.seqdb_v2.FastaUtil import FastaUtil
|
|
19
|
+
from wwpdb.utils.seqdb_v2.FetchUnpXml import FetchUnpXml
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class FetchUniProtEntry:
|
|
23
|
+
"""
|
|
24
|
+
Manage execution of fetch queries for UniProt entries.
|
|
25
|
+
|
|
26
|
+
This class wraps fetch data for UniProt entries including capturing
|
|
27
|
+
correct variant sequences from UniProt distributed Fasta file.
|
|
28
|
+
This avoids any inconsitency with Blast search results --
|
|
29
|
+
|
|
30
|
+
"""
|
|
31
|
+
|
|
32
|
+
def __init__(self, siteId=None, maxLength=100, verbose=False, log=sys.stderr):
|
|
33
|
+
self.__verbose = verbose
|
|
34
|
+
self.__siteId = siteId
|
|
35
|
+
self.__lfh = log
|
|
36
|
+
self.__maxLength = maxLength
|
|
37
|
+
self.__fXml = FetchUnpXml(maxLength=self.__maxLength, verbose=self.__verbose, log=self.__lfh)
|
|
38
|
+
self.__fFa = FastaUtil(verbose=self.__verbose, log=self.__lfh)
|
|
39
|
+
self.__vD = {}
|
|
40
|
+
self.__faLoaded = False
|
|
41
|
+
cICommon = ConfigInfoAppCommon(self.__siteId)
|
|
42
|
+
self.__fastaPath = cICommon.get_site_refdata_sequence_db_path()
|
|
43
|
+
self.__variantFastaFilePath = os.path.join(self.__fastaPath, "uniprot_sprot_varsplic.fasta")
|
|
44
|
+
|
|
45
|
+
def __loadVariants(self):
|
|
46
|
+
try:
|
|
47
|
+
_vList, self.__vD = self.__fFa.loadFastaUniProt(fastaFilePath=self.__variantFastaFilePath)
|
|
48
|
+
self.__faLoaded = True
|
|
49
|
+
except: # noqa: E722 pylint: disable=bare-except
|
|
50
|
+
if self.__verbose:
|
|
51
|
+
self.__lfh.write(
|
|
52
|
+
"+FetchUniprotEntry.__loadVariants() failed for file %s\n" % self.__variantFastaFilePath
|
|
53
|
+
)
|
|
54
|
+
traceback.print_exc(file=self.__lfh)
|
|
55
|
+
|
|
56
|
+
def fetchList(self, idList):
|
|
57
|
+
return self.__fXml.fetchList(idList)
|
|
58
|
+
|
|
59
|
+
def writeUnpXml(self, filename):
|
|
60
|
+
return self.__fXml.writeUnpXml(filename)
|
|
61
|
+
|
|
62
|
+
def getResult(self):
|
|
63
|
+
"""Get the parsed UniProt entry data in a dictionary structure.
|
|
64
|
+
|
|
65
|
+
Return dictionary has a key of input accession code which references
|
|
66
|
+
a dictionary containing the entry data.
|
|
67
|
+
|
|
68
|
+
"""
|
|
69
|
+
try:
|
|
70
|
+
rD = self.__fXml.getResult()
|
|
71
|
+
for uid, uD in rD.items():
|
|
72
|
+
if "db_isoform" in uD:
|
|
73
|
+
if self.__verbose:
|
|
74
|
+
self.__lfh.write("+FetchUniProtEntry.getResult() searching isoform %s\n" % uD["db_isoform"])
|
|
75
|
+
if not self.__faLoaded:
|
|
76
|
+
self.__loadVariants()
|
|
77
|
+
if uid in self.__vD:
|
|
78
|
+
uD["sequence"] = self.__vD[uid]["sequence"]
|
|
79
|
+
uD["db_isoform_description"] = self.__vD[uid]["description"]
|
|
80
|
+
uD["isoform_sequence_updated_from_fasta"] = "Y"
|
|
81
|
+
if self.__verbose:
|
|
82
|
+
self.__lfh.write(
|
|
83
|
+
"+FetchUniProtEntry.getResult() Updating isoform sequences from FASTA data for %s\n"
|
|
84
|
+
% uid
|
|
85
|
+
)
|
|
86
|
+
self.__lfh.write(
|
|
87
|
+
"+FetchUniProtEntry.getResult() Updating isoform description from FASTA %s\n"
|
|
88
|
+
% uD["db_isoform_description"]
|
|
89
|
+
)
|
|
90
|
+
return rD
|
|
91
|
+
except: # noqa: E722 pylint: disable=bare-except
|
|
92
|
+
traceback.print_exc(file=self.__lfh)
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def main(): # pragma: no cover
|
|
96
|
+
_funp = FetchUniProtEntry() # noqa: F841
|
|
@@ -0,0 +1,323 @@
|
|
|
1
|
+
##
|
|
2
|
+
# File: FetchUnpXml.py
|
|
3
|
+
# Author: Zukang Feng
|
|
4
|
+
# Update: 24-May-2010
|
|
5
|
+
# Version: 001 Initial version
|
|
6
|
+
#
|
|
7
|
+
# Update:
|
|
8
|
+
#
|
|
9
|
+
# 02-Jan-2013 jdw revise to support lists of accession codes as input.
|
|
10
|
+
# 03-Jan-2013 jdw revise variant handling.
|
|
11
|
+
# 13-Mar-2013 jdw Fix initialization issue
|
|
12
|
+
#
|
|
13
|
+
# 29-Jul-2014 jdw WARNING -- Handling of variant seqeuences is incomplete --
|
|
14
|
+
# 02-Sep-2020 zf added a work-around for isoforms sequence.
|
|
15
|
+
# 29-Dec-2020 zf added getMultipleResultDict()
|
|
16
|
+
# 25-Sep-2022 zf removed the work-around for isoforms sequence. It seems that the regular dbfetch tool can handle isoforms sequence now.
|
|
17
|
+
# 20-Nov-2023 zf fixed the issue related to handle isoforms sequences
|
|
18
|
+
##
|
|
19
|
+
|
|
20
|
+
__author__ = "Zukang Feng"
|
|
21
|
+
__email__ = "zfeng@rcsb.rutgers.edu"
|
|
22
|
+
__version__ = "V0.001"
|
|
23
|
+
|
|
24
|
+
import getopt
|
|
25
|
+
import sys
|
|
26
|
+
import traceback
|
|
27
|
+
|
|
28
|
+
import requests
|
|
29
|
+
import urllib3
|
|
30
|
+
|
|
31
|
+
try:
|
|
32
|
+
# Python 3
|
|
33
|
+
from itertools import zip_longest
|
|
34
|
+
except ImportError:
|
|
35
|
+
# Python 2
|
|
36
|
+
from itertools import izip_longest as zip_longest # type: ignore
|
|
37
|
+
|
|
38
|
+
import logging
|
|
39
|
+
|
|
40
|
+
from wwpdb.utils.config.ConfigInfo import ConfigInfo
|
|
41
|
+
from wwpdb.utils.seqdb_v2.ReadUnpXml import ReadUnpXmlString
|
|
42
|
+
|
|
43
|
+
logger = logging.getLogger()
|
|
44
|
+
|
|
45
|
+
# To disable warning about not checking ssl certificates. Still needed?
|
|
46
|
+
urllib3.disable_warnings()
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
class FetchUnpXml:
|
|
50
|
+
"""
|
|
51
|
+
Manage execution of fetch queries for UniProt entries.
|
|
52
|
+
|
|
53
|
+
XML entry data is parsed into a dictionary structure.
|
|
54
|
+
|
|
55
|
+
"""
|
|
56
|
+
|
|
57
|
+
def __init__(self, maxLength=100, verbose=False, log=sys.stderr):
|
|
58
|
+
self.__verbose = verbose
|
|
59
|
+
self.__debug = False
|
|
60
|
+
self.__lfh = log
|
|
61
|
+
self._baseUrl = "https://www.ebi.ac.uk/Tools/dbfetch/dbfetch"
|
|
62
|
+
self._baseUrlUnp = "https://www.ebi.ac.uk/proteins/api/proteins"
|
|
63
|
+
self.__maxLength = maxLength
|
|
64
|
+
self.__dataList = []
|
|
65
|
+
self.__result = {}
|
|
66
|
+
self.__multipleResult = {}
|
|
67
|
+
self.__idList = []
|
|
68
|
+
# self.__searchIdList = []
|
|
69
|
+
# self.__variantD = {}
|
|
70
|
+
#
|
|
71
|
+
# self.__isoformIdList = []
|
|
72
|
+
#
|
|
73
|
+
cI = ConfigInfo()
|
|
74
|
+
self.__forcefallback = cI.get("FETCH_UNP_FORCE_FALLBACK", None)
|
|
75
|
+
|
|
76
|
+
def fetchList(self, idList):
|
|
77
|
+
"""Execute a fetch query for the input id list.
|
|
78
|
+
The input list is filtered for variants (e.g. ids with appended '-#').
|
|
79
|
+
|
|
80
|
+
Divide the input list into manageable chunks, fetch each chunk,
|
|
81
|
+
and concatenate the result.
|
|
82
|
+
|
|
83
|
+
Return True for success or False otherwise.
|
|
84
|
+
|
|
85
|
+
"""
|
|
86
|
+
try:
|
|
87
|
+
if not idList:
|
|
88
|
+
return False
|
|
89
|
+
self.__result = {}
|
|
90
|
+
self.__multipleResult = {}
|
|
91
|
+
self.__dataList = []
|
|
92
|
+
self.__idList = []
|
|
93
|
+
# self.__isoformIdList = []
|
|
94
|
+
|
|
95
|
+
for accId in idList:
|
|
96
|
+
# splitList = accId.split("-")
|
|
97
|
+
# if len(splitList) == 2:
|
|
98
|
+
# if accId not in self.__isoformIdList:
|
|
99
|
+
# self.__isoformIdList.append(accId)
|
|
100
|
+
# #
|
|
101
|
+
# else:
|
|
102
|
+
# if accId not in self.__idList:
|
|
103
|
+
# self.__idList.append(accId)
|
|
104
|
+
# #
|
|
105
|
+
# #
|
|
106
|
+
if accId not in self.__idList:
|
|
107
|
+
self.__idList.append(accId)
|
|
108
|
+
|
|
109
|
+
# num = self.__processIdList()
|
|
110
|
+
if self.__verbose:
|
|
111
|
+
self.__lfh.write("+FetchUnpXml.fetchList() input id list len %d\n" % len(idList))
|
|
112
|
+
if self.__debug:
|
|
113
|
+
self.__lfh.write("+FetchUnpXml.fetchList() input id list %s\n" % idList)
|
|
114
|
+
# self.__lfh.write("+FetchUnpXml.fetchList() search list %s\n" % self.__searchIdList)
|
|
115
|
+
# self.__lfh.write("+FetchUnpXml.fetchList() variants %s\n" % self.__variantD.items())
|
|
116
|
+
# if num == 0: # and (not self.__isoformIdList):
|
|
117
|
+
# return False
|
|
118
|
+
# #
|
|
119
|
+
# if num:
|
|
120
|
+
# subLists = self.__makeSubLists(self.__maxLength, self.__searchIdList)
|
|
121
|
+
if True: # pylint: disable=using-constant-test
|
|
122
|
+
subLists = self.__makeSubLists(self.__maxLength, self.__idList)
|
|
123
|
+
|
|
124
|
+
for subList in subLists:
|
|
125
|
+
idString = ",".join(subList)
|
|
126
|
+
if self.__debug:
|
|
127
|
+
self.__lfh.write("+FetchUnpXml.fetchList() subList %s string %s\n" % (subList, idString))
|
|
128
|
+
#
|
|
129
|
+
# If primary site has issues, automatic fallback
|
|
130
|
+
if self.__forcefallback:
|
|
131
|
+
xmlText = self.__RequestUnpXml(idString, fallback=True)
|
|
132
|
+
else:
|
|
133
|
+
try:
|
|
134
|
+
xmlText = self.__RequestUnpXml(idString)
|
|
135
|
+
except requests.exceptions.HTTPError:
|
|
136
|
+
xmlText = self.__RequestUnpXml(idString, fallback=True)
|
|
137
|
+
#
|
|
138
|
+
# filter possible simple text error messages from the failed queries.
|
|
139
|
+
if (xmlText is not None) and not xmlText.startswith("ERROR"):
|
|
140
|
+
if xmlText.find("</uniprot>\n") >= 0:
|
|
141
|
+
xmlTextList = xmlText.split("</uniprot>\n")
|
|
142
|
+
for xmlTxt in xmlTextList:
|
|
143
|
+
xmltxtstrip = xmlTxt.strip()
|
|
144
|
+
if xmltxtstrip.startswith("<?xml version="):
|
|
145
|
+
if xmltxtstrip.endswith("</uniprot>"):
|
|
146
|
+
self.__dataList.append(xmltxtstrip)
|
|
147
|
+
else:
|
|
148
|
+
self.__dataList.append(xmltxtstrip + "</uniprot>")
|
|
149
|
+
elif xmlText.find("</uniprot>") >= 0:
|
|
150
|
+
xmlTextList = xmlText.split("</uniprot>")
|
|
151
|
+
for xmlTxt in xmlTextList:
|
|
152
|
+
xmltxtstrip = xmlTxt.strip()
|
|
153
|
+
if xmltxtstrip.startswith("<?xml version="):
|
|
154
|
+
if xmltxtstrip.endswith("</uniprot>"):
|
|
155
|
+
self.__dataList.append(xmltxtstrip)
|
|
156
|
+
else:
|
|
157
|
+
self.__dataList.append(xmltxtstrip + "</uniprot>")
|
|
158
|
+
else:
|
|
159
|
+
self.__dataList.append(xmlText)
|
|
160
|
+
#
|
|
161
|
+
# for accId in self.__isoformIdList:
|
|
162
|
+
# xmlText = self.__RequestIsoformsUnpXml(accId)
|
|
163
|
+
# if xmlText:
|
|
164
|
+
# self.__dataList.append(xmlText)
|
|
165
|
+
# #
|
|
166
|
+
# #
|
|
167
|
+
if len(self.__dataList) > 0:
|
|
168
|
+
ok = self.__ParseUnpXmlData()
|
|
169
|
+
else:
|
|
170
|
+
ok = False
|
|
171
|
+
return ok
|
|
172
|
+
except Exception as e: # noqa: BLE001
|
|
173
|
+
if self.__verbose:
|
|
174
|
+
self.__lfh.write("+FetchUnpXml.fetchList() exception %s\n" % str(e))
|
|
175
|
+
traceback.print_exc(file=self.__lfh)
|
|
176
|
+
return False
|
|
177
|
+
|
|
178
|
+
def writeUnpXml(self, filename):
|
|
179
|
+
wfile = open(filename, "w")
|
|
180
|
+
for data in self.__dataList:
|
|
181
|
+
wfile.write(data)
|
|
182
|
+
wfile.close()
|
|
183
|
+
|
|
184
|
+
def getResult(self):
|
|
185
|
+
"""Get the parsed UniProt entry data in a dictionary structure.
|
|
186
|
+
|
|
187
|
+
Return dictionary has a key of input accession code which references
|
|
188
|
+
a dictionary containing the entry data.
|
|
189
|
+
|
|
190
|
+
"""
|
|
191
|
+
return self.__result
|
|
192
|
+
|
|
193
|
+
def getMultipleResultDict(self):
|
|
194
|
+
return self.__multipleResult
|
|
195
|
+
|
|
196
|
+
# def __processIdList(self):
|
|
197
|
+
# """
|
|
198
|
+
# Filter the input id list for variants and create the list of unique
|
|
199
|
+
# searchable id codes. Create a dictionary of variant identifiers and
|
|
200
|
+
# their corresponding searchable id codes.
|
|
201
|
+
|
|
202
|
+
# """
|
|
203
|
+
# self.__searchIdList = []
|
|
204
|
+
# self.__variantD = {}
|
|
205
|
+
# tList = []
|
|
206
|
+
# #
|
|
207
|
+
# for vid in self.__idList:
|
|
208
|
+
# # check for variant id
|
|
209
|
+
# idx = vid.find("-")
|
|
210
|
+
# if idx == -1:
|
|
211
|
+
# sId = vid
|
|
212
|
+
# else:
|
|
213
|
+
# sId = vid[0:idx]
|
|
214
|
+
# self.__variantD[vid] = sId
|
|
215
|
+
# #
|
|
216
|
+
# tList.append(sId)
|
|
217
|
+
# #
|
|
218
|
+
# # unique list of searchable accessions
|
|
219
|
+
# #
|
|
220
|
+
# self.__searchIdList = list(set(tList))
|
|
221
|
+
# #
|
|
222
|
+
# return len(self.__searchIdList)
|
|
223
|
+
|
|
224
|
+
@staticmethod
|
|
225
|
+
def __makeSubLists(n, iterable):
|
|
226
|
+
args = [iter(iterable)] * n
|
|
227
|
+
return ([e for e in t if e is not None] for t in zip_longest(*args))
|
|
228
|
+
|
|
229
|
+
# def __makeSubListsWithPadding(self, n, iterable, padvalue=None):
|
|
230
|
+
# "__sublist(3, 'abcdefg', 'x') --> ('a','b','c'), ('d','e','f'), ('g','x','x')"
|
|
231
|
+
# return zip_longest(*[iter(iterable)] * n, fillvalue=padvalue)
|
|
232
|
+
|
|
233
|
+
def __RequestUnpXml(self, idString, fallback=False):
|
|
234
|
+
"""Execute fetch Request for the input comma separated accession list.
|
|
235
|
+
|
|
236
|
+
Return xml text for the corresentry UniProt entries
|
|
237
|
+
"""
|
|
238
|
+
if not fallback:
|
|
239
|
+
params = {}
|
|
240
|
+
params["db"] = "uniprotkb"
|
|
241
|
+
params["id"] = idString
|
|
242
|
+
|
|
243
|
+
params["format"] = "uniprotxml"
|
|
244
|
+
params["style"] = "raw"
|
|
245
|
+
# params['Retrieve'] = 'Retrieve'
|
|
246
|
+
logger.debug("Request %s with data %s", self._baseUrl, params)
|
|
247
|
+
reqH = requests.post(self._baseUrl, data=params, verify=False, timeout=30) # noqa: S501
|
|
248
|
+
reqH.raise_for_status()
|
|
249
|
+
else:
|
|
250
|
+
params = {}
|
|
251
|
+
params["size"] = "-1"
|
|
252
|
+
params["accession"] = idString
|
|
253
|
+
|
|
254
|
+
# Need to do this as UNP service will not take POST - so force GET
|
|
255
|
+
logger.debug("Request with data %s", params)
|
|
256
|
+
reqH = requests.get(
|
|
257
|
+
self._baseUrlUnp, params=params, headers={"Accept": "application/xml"}, verify=False, timeout=30 # noqa: S501
|
|
258
|
+
)
|
|
259
|
+
reqH.raise_for_status()
|
|
260
|
+
#
|
|
261
|
+
# data = reqH.read()
|
|
262
|
+
data = reqH.text
|
|
263
|
+
return data
|
|
264
|
+
|
|
265
|
+
# def __RequestIsoformsUnpXml(self, accId):
|
|
266
|
+
# """Using URL https://www.ebi.ac.uk/proteins/api/proteins/{accession}/isoforms"""
|
|
267
|
+
# isoformUrl = self._baseUrlUnp + "/" + accId + "/isoforms"
|
|
268
|
+
# params = {}
|
|
269
|
+
# reqH = requests.get(isoformUrl, params=params, headers={"Accept": "application/xml"}, verify=False, timeout=30)
|
|
270
|
+
# reqH.raise_for_status()
|
|
271
|
+
# data = reqH.text
|
|
272
|
+
# if data.find("errorMessages") >= 0:
|
|
273
|
+
# return ""
|
|
274
|
+
# #
|
|
275
|
+
# return data
|
|
276
|
+
|
|
277
|
+
def __ParseUnpXmlData(self):
|
|
278
|
+
"""
|
|
279
|
+
Parse the accumulated xml text for each chunk and store the parsed data in
|
|
280
|
+
the internal result dictionary.
|
|
281
|
+
|
|
282
|
+
variant id codes are regisistered with the parser so that the returned dictionary
|
|
283
|
+
contains keys corresponding to the original input id list.
|
|
284
|
+
|
|
285
|
+
"""
|
|
286
|
+
try:
|
|
287
|
+
self.__result = {}
|
|
288
|
+
self.__multipleResult = {}
|
|
289
|
+
for data in self.__dataList:
|
|
290
|
+
readxml = ReadUnpXmlString(data, verbose=self.__verbose, log=self.__lfh)
|
|
291
|
+
# for vId, aId in self.__variantD.items():
|
|
292
|
+
# readxml.addVariant(aId, vId)
|
|
293
|
+
# #
|
|
294
|
+
self.__result.update(readxml.getResult())
|
|
295
|
+
self.__multipleResult.update(readxml.getMultipleResultDict())
|
|
296
|
+
return True
|
|
297
|
+
except Exception as e: # noqa: BLE001
|
|
298
|
+
if self.__verbose:
|
|
299
|
+
self.__lfh.write("+FetchUnpXml.__ParseUnpXmlData() exception %s\n" % str(e))
|
|
300
|
+
traceback.print_exc(file=self.__lfh)
|
|
301
|
+
return False
|
|
302
|
+
|
|
303
|
+
|
|
304
|
+
def main(argv): # pragma: no cover
|
|
305
|
+
opts, _args = getopt.getopt(argv, "i:", ["id="])
|
|
306
|
+
for opt, arg in opts:
|
|
307
|
+
if opt in ("-i", "--id"):
|
|
308
|
+
uid = arg
|
|
309
|
+
fobj = FetchUnpXml(verbose=True)
|
|
310
|
+
fobj.fetchList([uid])
|
|
311
|
+
fobj.writeUnpXml(uid + ".xml")
|
|
312
|
+
rdict = fobj.getResult()
|
|
313
|
+
for k, v in rdict.items():
|
|
314
|
+
sys.stdout.write("%-30s = %s\n" % (k, v))
|
|
315
|
+
|
|
316
|
+
|
|
317
|
+
if __name__ == "__main__": # pragma: no cover
|
|
318
|
+
try:
|
|
319
|
+
main(sys.argv[1:])
|
|
320
|
+
sys.exit(0)
|
|
321
|
+
except Exception as exc: # noqa: BLE001
|
|
322
|
+
sys.stderr.write("Exception: %s\n" % str(exc))
|
|
323
|
+
sys.exit(1)
|