rcsb.utils.chemref 0.96__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- rcsb/__init__.py +1 -0
- rcsb/utils/__init__.py +1 -0
- rcsb/utils/chemref/AtcProvider.py +269 -0
- rcsb/utils/chemref/BirdProvider.py +158 -0
- rcsb/utils/chemref/CARDProvider.py +102 -0
- rcsb/utils/chemref/CODProvider.py +132 -0
- rcsb/utils/chemref/ChEMBLProvider.py +269 -0
- rcsb/utils/chemref/ChemCompModelProvider.py +196 -0
- rcsb/utils/chemref/ChemCompProvider.py +289 -0
- rcsb/utils/chemref/DrugBankProvider.py +408 -0
- rcsb/utils/chemref/DrugBankReader.py +220 -0
- rcsb/utils/chemref/DrugCentralProvider.py +113 -0
- rcsb/utils/chemref/PharosProvider.py +108 -0
- rcsb/utils/chemref/PharosReadSqlDump.py +76 -0
- rcsb/utils/chemref/Pharos_dump_tables.sh +7 -0
- rcsb/utils/chemref/PsiModProvider.py +219 -0
- rcsb/utils/chemref/PubChemProvider.py +117 -0
- rcsb/utils/chemref/PubChemUtils.py +1173 -0
- rcsb/utils/chemref/RcsbLigandScoreProvider.py +146 -0
- rcsb/utils/chemref/ResidProvider.py +177 -0
- rcsb/utils/chemref/ResidReader.py +46 -0
- rcsb/utils/chemref/__init__.py +0 -0
- rcsb_utils_chemref-0.96.dist-info/METADATA +66 -0
- rcsb_utils_chemref-0.96.dist-info/RECORD +26 -0
- rcsb_utils_chemref-0.96.dist-info/WHEEL +4 -0
- rcsb_utils_chemref-0.96.dist-info/licenses/LICENSE +12 -0
rcsb/__init__.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__path__ = __import__("pkgutil").extend_path(__path__, __name__)
|
rcsb/utils/__init__.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__path__ = __import__("pkgutil").extend_path(__path__, __name__)
|
|
@@ -0,0 +1,269 @@
|
|
|
1
|
+
##
|
|
2
|
+
# File: AtcProvider.py
|
|
3
|
+
# Date: 3-Apr-2019 jdw
|
|
4
|
+
#
|
|
5
|
+
# Updates:
|
|
6
|
+
# 5-Jun-2021 jdw Update ATC data source and fallback
|
|
7
|
+
# 20-Jul-2021 jdw Make this provider a subclass of StashableBase
|
|
8
|
+
# 3-Jan-2022 dwp Update data loading methods to address recent changes in source NCBO ATC files
|
|
9
|
+
# 25-Jul-2022 dwp Revert last change - source NCBO ATC files were updated again to restore previous format
|
|
10
|
+
# 27-Jul-2022 dwp Adapt to possible naming schemes for class IDs (may be identified with "ATC" or "UATC")
|
|
11
|
+
# 22-Oct-2024 dwp Add standardization method for ATC data to handle potential stringified arrays
|
|
12
|
+
##
|
|
13
|
+
"""
|
|
14
|
+
Extract ATC term descriptions from NCBO ATC flat files.
|
|
15
|
+
|
|
16
|
+
"""
|
|
17
|
+
|
|
18
|
+
import collections
|
|
19
|
+
import logging
|
|
20
|
+
import os.path
|
|
21
|
+
import sys
|
|
22
|
+
import ast
|
|
23
|
+
|
|
24
|
+
from rcsb.utils.io.FileUtil import FileUtil
|
|
25
|
+
from rcsb.utils.io.MarshalUtil import MarshalUtil
|
|
26
|
+
from rcsb.utils.io.StashableBase import StashableBase
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
logger = logging.getLogger(__name__)
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
class AtcProvider(StashableBase):
|
|
33
|
+
"""Extract term descriptions and ATC classifications from ATC flat files."""
|
|
34
|
+
|
|
35
|
+
def __init__(self, **kwargs):
|
|
36
|
+
self.__dirName = "atc"
|
|
37
|
+
self.__cachePath = kwargs.get("cachePath", ".")
|
|
38
|
+
super(AtcProvider, self).__init__(self.__cachePath, [self.__dirName])
|
|
39
|
+
atcDirPath = os.path.join(self.__cachePath, self.__dirName)
|
|
40
|
+
useCache = kwargs.get("useCache", True)
|
|
41
|
+
|
|
42
|
+
urlTarget = "https://data.bioontology.org/ontologies/ATC/download?apikey=8b5b7825-538d-40e0-9e9e-5ab9274a9aeb&download_format=csv"
|
|
43
|
+
urlTargetFallback = "https://github.com/rcsb/py-rcsb_exdb_assets/raw/master/fall_back/ATC.csv.gz"
|
|
44
|
+
self.__version = kwargs.get("AtcVersion", "2023")
|
|
45
|
+
#
|
|
46
|
+
self.__mU = MarshalUtil(workPath=atcDirPath)
|
|
47
|
+
self.__atcD = self.__reload(urlTarget, urlTargetFallback, atcDirPath, useCache=useCache)
|
|
48
|
+
#
|
|
49
|
+
|
|
50
|
+
def getVersion(self):
|
|
51
|
+
return self.__version
|
|
52
|
+
|
|
53
|
+
def __reload(self, urlTarget, urlTargetFallback, atcDirPath, useCache=True):
|
|
54
|
+
pyVersion = sys.version_info[0]
|
|
55
|
+
atcFilePath = os.path.join(atcDirPath, "atc-py%s.pic" % str(pyVersion))
|
|
56
|
+
#
|
|
57
|
+
atcD = {}
|
|
58
|
+
if useCache and self.__mU.exists(atcFilePath):
|
|
59
|
+
atcD = self.__mU.doImport(atcFilePath, fmt="pickle")
|
|
60
|
+
# logger.debug("ATC name length %d parent length %d assignments %d", len(atcD["names"]), len(atcD["parents"]), len(atcD["assignments"]))
|
|
61
|
+
# nD = atcD["names"]
|
|
62
|
+
# pD = atcD["parents"]
|
|
63
|
+
|
|
64
|
+
elif not useCache:
|
|
65
|
+
fn = "ATC.csv.gz"
|
|
66
|
+
fp = os.path.join(atcDirPath, fn)
|
|
67
|
+
logger.debug("Fetch ATC term descriptions from source %s", fp)
|
|
68
|
+
fileU = FileUtil(workPath=atcDirPath)
|
|
69
|
+
fileU.mkdir(atcDirPath)
|
|
70
|
+
try:
|
|
71
|
+
ok = fileU.get(urlTarget, fp)
|
|
72
|
+
logger.info("ATC fetch status is %r", ok)
|
|
73
|
+
if not ok:
|
|
74
|
+
ok = fileU.get(urlTargetFallback, fp)
|
|
75
|
+
logger.info("ATC fallback fetch status is %r", ok)
|
|
76
|
+
atcL = self.__mU.doImport(fp, fmt="csv", rowFormat="dict", uncomment=False)
|
|
77
|
+
#
|
|
78
|
+
columnL = list(atcL[0].keys()) if atcL else []
|
|
79
|
+
nD = self.__extractNames(atcL)
|
|
80
|
+
pD = self.__extractHierarchy(atcL)
|
|
81
|
+
atcD = {"names": nD, "parents": pD}
|
|
82
|
+
ok = self.__mU.doExport(atcFilePath, atcD, fmt="pickle")
|
|
83
|
+
logger.info("ATC cache status %r data length %d columns %r names %d parents %d", ok, len(atcL), columnL, len(nD), len(pD))
|
|
84
|
+
except Exception as e:
|
|
85
|
+
logger.exception("Failing for %r with %s", fp, str(e))
|
|
86
|
+
#
|
|
87
|
+
return atcD
|
|
88
|
+
|
|
89
|
+
def getAtcVersion(self):
|
|
90
|
+
return self.__version
|
|
91
|
+
|
|
92
|
+
def testCache(self):
|
|
93
|
+
if self.__atcD and "names" in self.__atcD and "parents" in self.__atcD and (len(self.__atcD["names"]) > 6100) and (len(self.__atcD["parents"]) > 6100):
|
|
94
|
+
logger.info("ATC names %d parents %d", len(self.__atcD["names"]), len(self.__atcD["parents"]))
|
|
95
|
+
return True
|
|
96
|
+
return False
|
|
97
|
+
|
|
98
|
+
def getAtcName(self, atcId):
|
|
99
|
+
try:
|
|
100
|
+
return self.__atcD["names"][atcId]["name"]
|
|
101
|
+
except Exception:
|
|
102
|
+
logger.info("Undefined ATC %r", atcId)
|
|
103
|
+
return None
|
|
104
|
+
|
|
105
|
+
def getIdLineage(self, atcId):
|
|
106
|
+
pList = []
|
|
107
|
+
try:
|
|
108
|
+
pList.append(atcId)
|
|
109
|
+
pt = self.__atcD["parents"][atcId]
|
|
110
|
+
while (pt is not None) and (pt != 0):
|
|
111
|
+
pList.append(pt)
|
|
112
|
+
pt = self.__atcD["parents"][pt]
|
|
113
|
+
except Exception as e:
|
|
114
|
+
logger.exception("Failing for %r with %s", atcId, str(e))
|
|
115
|
+
#
|
|
116
|
+
pList.reverse()
|
|
117
|
+
return pList
|
|
118
|
+
|
|
119
|
+
def getNameLineage(self, atcId):
|
|
120
|
+
try:
|
|
121
|
+
return [self.getAtcName(cId) for cId in self.getIdLineage(atcId)]
|
|
122
|
+
except Exception as e:
|
|
123
|
+
logger.exception("Failing for %r with %s", atcId, str(e))
|
|
124
|
+
return None
|
|
125
|
+
|
|
126
|
+
def getTreeNodeList(self, filterD=None):
|
|
127
|
+
return self.__exportTreeNodeList(self.__atcD["names"], self.__atcD["parents"], filterD=filterD)
|
|
128
|
+
|
|
129
|
+
def __extractNames(self, atcL):
|
|
130
|
+
"""
|
|
131
|
+
Example public data set from BioPortal -
|
|
132
|
+
Class ID,Preferred Label,Synonyms,Definitions,Obsolete,CUI,Semantic Types,Parents,ATC LEVEL,Is Drug Class,Semantic type UMLS property
|
|
133
|
+
|
|
134
|
+
http://purl.bioontology.org/ontology/ATC/A03AX13,silicones,,,false,C0037114,http://purl.bioontology.org/ontology/STY/T109|http://purl.bioontology.org/ontology/STY/T122,http://purl.bioontology.org/ontology/UATC/A03AX,5,,http://purl.bioontology.org/ontology/STY/T109|http://purl.bioontology.org/ontology/STY/T122
|
|
135
|
+
http://purl.bioontology.org/ontology/ATC/J01DB07,cefatrizine,,,false,C0007545,http://purl.bioontology.org/ontology/STY/T195|http://purl.bioontology.org/ontology/STY/T109,http://purl.bioontology.org/ontology/UATC/J01DB,5,,http://purl.bioontology.org/ontology/STY/T195|http://purl.bioontology.org/ontology/STY/T109
|
|
136
|
+
|
|
137
|
+
Occasionally changes from "ATC" to "UATC", so capture these too:
|
|
138
|
+
http://purl.bioontology.org/ontology/UATC/A03AX13,silicones,,,false,C0037114,http://purl.bioontology.org/ontology/STY/T109|http://purl.bioontology.org/ontology/STY/T122,http://purl.bioontology.org/ontology/UATC/A03AX,5,,http://purl.bioontology.org/ontology/STY/T109|http://purl.bioontology.org/ontology/STY/T122
|
|
139
|
+
http://purl.bioontology.org/ontology/UATC/J01DB07,cefatrizine,,,false,C0007545,http://purl.bioontology.org/ontology/STY/T195|http://purl.bioontology.org/ontology/STY/T109,http://purl.bioontology.org/ontology/UATC/J01DB,5,,http://purl.bioontology.org/ontology/STY/T195|http://purl.bioontology.org/ontology/STY/T109
|
|
140
|
+
"""
|
|
141
|
+
nD = {}
|
|
142
|
+
nsATC = "http://purl.bioontology.org/ontology/ATC/"
|
|
143
|
+
nsUATC = "http://purl.bioontology.org/ontology/UATC/"
|
|
144
|
+
for rD in atcL:
|
|
145
|
+
if nsATC in rD["Class ID"]:
|
|
146
|
+
ns = nsATC
|
|
147
|
+
elif nsUATC in rD["Class ID"]:
|
|
148
|
+
ns = nsUATC
|
|
149
|
+
else:
|
|
150
|
+
continue
|
|
151
|
+
idCode = rD["Class ID"].replace(ns, "")
|
|
152
|
+
name = self.__standardizePreferredLabel(rD["Preferred Label"])
|
|
153
|
+
synonyms = rD["Synonyms"]
|
|
154
|
+
definition = rD["Definitions"]
|
|
155
|
+
level = int(rD["ATC LEVEL"]) if rD["ATC LEVEL"].isdigit() else None
|
|
156
|
+
nD[idCode] = {"name": name, "synonyms": synonyms, "definition": definition, "level": level}
|
|
157
|
+
logger.info("Length of name dictionary %d", len(nD))
|
|
158
|
+
return nD
|
|
159
|
+
|
|
160
|
+
def __standardizePreferredLabel(self, label):
|
|
161
|
+
"""Temporary method to standardize the "Preferred Label" value of an ATC datum,
|
|
162
|
+
due to recent unintended changes to source ATC data in which the column is appearing
|
|
163
|
+
as a stringified array instead of a simple string. E.g.,:
|
|
164
|
+
|
|
165
|
+
Before:
|
|
166
|
+
http://purl.bioontology.org/ontology/ATC/B05DB,Hypertonic solutions,...<remainder_omitted>
|
|
167
|
+
|
|
168
|
+
Now (as of Version 2024AA):
|
|
169
|
+
http://purl.bioontology.org/ontology/ATC/B05DB,"[""Hypertonic solutions""]",...<remainder_omitted>
|
|
170
|
+
|
|
171
|
+
Once this is resolved at the source data level, can stop using this method.
|
|
172
|
+
|
|
173
|
+
Args:
|
|
174
|
+
label (str): raw label
|
|
175
|
+
|
|
176
|
+
Returns:
|
|
177
|
+
label: processed label
|
|
178
|
+
"""
|
|
179
|
+
try:
|
|
180
|
+
# Attempt to parse the label as a list
|
|
181
|
+
parsedLabel = ast.literal_eval(label)
|
|
182
|
+
# If it's a list, return the first element (assuming it's a single-item array)
|
|
183
|
+
if isinstance(parsedLabel, list) and len(parsedLabel) == 1:
|
|
184
|
+
return parsedLabel[0]
|
|
185
|
+
else:
|
|
186
|
+
logger.info("parsed label %r", parsedLabel)
|
|
187
|
+
except (ValueError, SyntaxError):
|
|
188
|
+
# If not a list or parsing fails, return the original label
|
|
189
|
+
return label
|
|
190
|
+
return label
|
|
191
|
+
|
|
192
|
+
def __extractHierarchy(self, atcL):
|
|
193
|
+
""" """
|
|
194
|
+
pD = {}
|
|
195
|
+
nsATC = "http://purl.bioontology.org/ontology/ATC/"
|
|
196
|
+
nsUATC = "http://purl.bioontology.org/ontology/UATC/"
|
|
197
|
+
for rD in atcL:
|
|
198
|
+
if nsATC in rD["Class ID"]:
|
|
199
|
+
ns = nsATC
|
|
200
|
+
elif nsUATC in rD["Class ID"]:
|
|
201
|
+
ns = nsUATC
|
|
202
|
+
else:
|
|
203
|
+
continue
|
|
204
|
+
idCode = rD["Class ID"].replace(ns, "")
|
|
205
|
+
pIdCode = rD["Parents"].replace(ns, "")
|
|
206
|
+
pIdCode = None if "http://www.w3.org/2002/07/owl#Thing" in pIdCode else pIdCode
|
|
207
|
+
pD[idCode] = pIdCode
|
|
208
|
+
#
|
|
209
|
+
logger.info("Length of parent dictionary %d", len(pD))
|
|
210
|
+
# for ky, vl in pD.items():
|
|
211
|
+
# logger.info(" %r %r", ky, vl)
|
|
212
|
+
return pD
|
|
213
|
+
|
|
214
|
+
def __exportTreeNodeList(self, nD, pD, filterD=None):
|
|
215
|
+
"""Create node list from the ATC parent and name/description dictionaries."""
|
|
216
|
+
#
|
|
217
|
+
# pL = [0]
|
|
218
|
+
pL = []
|
|
219
|
+
logger.info("nD %d pD %d", len(nD), len(pD))
|
|
220
|
+
# create child dictionary
|
|
221
|
+
cD = {}
|
|
222
|
+
for ctId, ptId in pD.items():
|
|
223
|
+
if not ptId:
|
|
224
|
+
pL.append(ctId)
|
|
225
|
+
continue
|
|
226
|
+
cD.setdefault(ptId, []).append(ctId)
|
|
227
|
+
|
|
228
|
+
#
|
|
229
|
+
logger.debug("cD %d", len(cD))
|
|
230
|
+
#
|
|
231
|
+
idL = []
|
|
232
|
+
for rootId in sorted(pL):
|
|
233
|
+
visited = set([rootId])
|
|
234
|
+
queue = collections.deque(visited)
|
|
235
|
+
while queue:
|
|
236
|
+
tId = queue.popleft()
|
|
237
|
+
idL.append(tId)
|
|
238
|
+
if tId not in cD:
|
|
239
|
+
# logger.warning("No children for Atc tId %r" % tId)
|
|
240
|
+
continue
|
|
241
|
+
for childId in cD[tId]:
|
|
242
|
+
if childId not in visited:
|
|
243
|
+
queue.append(childId)
|
|
244
|
+
visited.add(childId)
|
|
245
|
+
#
|
|
246
|
+
uD = {}
|
|
247
|
+
dL = []
|
|
248
|
+
for tId in idL:
|
|
249
|
+
if filterD and tId not in filterD:
|
|
250
|
+
continue
|
|
251
|
+
# id plus parents
|
|
252
|
+
idlL = self.getIdLineage(tId)
|
|
253
|
+
for iDepth, idl in enumerate(idlL, 0):
|
|
254
|
+
if idl in uD:
|
|
255
|
+
continue
|
|
256
|
+
uD[idl] = True
|
|
257
|
+
displayName = nD[idl]["name"] if idl in nD else None
|
|
258
|
+
ptId = pD[idl] if idl in pD else None
|
|
259
|
+
#
|
|
260
|
+
# dD = {"id": tId, "name": displayName, "lineage": lL, "parents": [ptId], "depth": len(lL)}
|
|
261
|
+
#
|
|
262
|
+
if ptId:
|
|
263
|
+
dD = {"id": idl, "name": displayName, "parents": [ptId], "depth": iDepth}
|
|
264
|
+
else:
|
|
265
|
+
dD = {"id": idl, "name": displayName, "depth": iDepth}
|
|
266
|
+
#
|
|
267
|
+
dL.append(dD)
|
|
268
|
+
|
|
269
|
+
return dL
|
|
@@ -0,0 +1,158 @@
|
|
|
1
|
+
##
|
|
2
|
+
# File: BirdProvider.py
|
|
3
|
+
# Author: J. Westbrook
|
|
4
|
+
# Date: 1-Jun-2021
|
|
5
|
+
#
|
|
6
|
+
# Updates:
|
|
7
|
+
# 21-Jul-2021 jdw Make this provider a subclass of StashableBase
|
|
8
|
+
##
|
|
9
|
+
"""
|
|
10
|
+
Utilities to read and serialize portions of the dictionary of PDBx/mmCIF BIRD definitions.
|
|
11
|
+
"""
|
|
12
|
+
__docformat__ = "google en"
|
|
13
|
+
__author__ = "John Westbrook"
|
|
14
|
+
__email__ = "john.westbrook@rcsb.org"
|
|
15
|
+
__license__ = "Apache 2.0"
|
|
16
|
+
|
|
17
|
+
import logging
|
|
18
|
+
import os
|
|
19
|
+
import time
|
|
20
|
+
|
|
21
|
+
from rcsb.utils.io.FileUtil import FileUtil
|
|
22
|
+
from rcsb.utils.io.MarshalUtil import MarshalUtil
|
|
23
|
+
from rcsb.utils.io.StashableBase import StashableBase
|
|
24
|
+
|
|
25
|
+
logger = logging.getLogger(__name__)
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
class BirdProvider(StashableBase):
|
|
29
|
+
"""Utilities to read and serialize portions of the dictionary of PDBx/mmCIF BIRD definitions."""
|
|
30
|
+
|
|
31
|
+
def __init__(self, **kwargs):
|
|
32
|
+
dirName = "bird"
|
|
33
|
+
cachePath = kwargs.get("cachePath", ".")
|
|
34
|
+
super(BirdProvider, self).__init__(cachePath, [dirName])
|
|
35
|
+
|
|
36
|
+
# Default source target locators
|
|
37
|
+
self.__birdUrlTarget = kwargs.get("birdUrlTarget", None)
|
|
38
|
+
self.__birdUrlTarget = self.__birdUrlTarget if self.__birdUrlTarget else "http://files.wwpdb.org/pub/pdb/data/bird/prd/prd-all.cif.gz"
|
|
39
|
+
#
|
|
40
|
+
dirPath = os.path.join(cachePath, dirName)
|
|
41
|
+
useCache = kwargs.get("useCache", True)
|
|
42
|
+
molLimit = kwargs.get("molLimit", 0)
|
|
43
|
+
#
|
|
44
|
+
self.__mU = MarshalUtil(workPath=dirPath)
|
|
45
|
+
self.__birdD = self.__reload(self.__birdUrlTarget, dirPath, useCache=useCache, molLimit=molLimit)
|
|
46
|
+
|
|
47
|
+
def testCache(self, minCount=1):
|
|
48
|
+
if self.__birdD and len(self.__birdD) >= minCount:
|
|
49
|
+
logger.info("BIRD definitions (%d)", len(self.__birdD))
|
|
50
|
+
return True
|
|
51
|
+
return False
|
|
52
|
+
|
|
53
|
+
def getBirdD(self):
|
|
54
|
+
return self.__birdD
|
|
55
|
+
|
|
56
|
+
def getRepresentation(self, prdId):
|
|
57
|
+
try:
|
|
58
|
+
return self.__birdD[prdId]["representAs"]
|
|
59
|
+
except Exception as e:
|
|
60
|
+
logger.debug("Get representation %r failing with %s", prdId, str(e))
|
|
61
|
+
return None
|
|
62
|
+
|
|
63
|
+
def getReleaseStatus(self, prdId):
|
|
64
|
+
try:
|
|
65
|
+
return self.__birdD[prdId]["releaseStatus"]
|
|
66
|
+
except Exception as e:
|
|
67
|
+
logger.debug("Get status %r failing with %s", prdId, str(e))
|
|
68
|
+
return None
|
|
69
|
+
|
|
70
|
+
def getChemCompId(self, prdId):
|
|
71
|
+
try:
|
|
72
|
+
return self.__birdD[prdId]["chemCompId"]
|
|
73
|
+
except Exception as e:
|
|
74
|
+
logger.debug("Get compId %r failing with %s", prdId, str(e))
|
|
75
|
+
return None
|
|
76
|
+
|
|
77
|
+
def getClassType(self, prdId):
|
|
78
|
+
try:
|
|
79
|
+
return self.__birdD[prdId]["class"]
|
|
80
|
+
except Exception as e:
|
|
81
|
+
logger.debug("Get class %r failing with %s", prdId, str(e))
|
|
82
|
+
return None
|
|
83
|
+
|
|
84
|
+
def __reload(self, birdUrlTarget, dirPath, useCache=False, molLimit=None):
|
|
85
|
+
"""Reload or create serialized data dictionary of BIRD molecule definitions.
|
|
86
|
+
|
|
87
|
+
Args:
|
|
88
|
+
birdUrlTarget (str): target url for bird dictionary resource file
|
|
89
|
+
dirPath (str): path to the directory containing cache files
|
|
90
|
+
useCache (bool):
|
|
91
|
+
molLimit (int): maximum number of definitions to process
|
|
92
|
+
|
|
93
|
+
Returns:
|
|
94
|
+
(list): list of bird data containers
|
|
95
|
+
"""
|
|
96
|
+
#
|
|
97
|
+
birdObjD = {}
|
|
98
|
+
startTime = time.time()
|
|
99
|
+
birdDataFilePath = os.path.join(dirPath, "bird-definitions.json")
|
|
100
|
+
_, fExt = os.path.splitext(birdDataFilePath)
|
|
101
|
+
birdDataFormat = "json" if fExt == ".json" else "pickle"
|
|
102
|
+
#
|
|
103
|
+
if useCache and self.__mU.exists(birdDataFilePath):
|
|
104
|
+
birdObjD = self.__mU.doImport(birdDataFilePath, fmt=birdDataFormat)
|
|
105
|
+
elif not useCache:
|
|
106
|
+
# Source component data files ...
|
|
107
|
+
birdFilePath = self.__fetchUrl(birdUrlTarget, dirPath, useCache=useCache)
|
|
108
|
+
birdObjD = self.__readBirdDefinitions(birdFilePath, molLimit=molLimit)
|
|
109
|
+
ok = self.__mU.doExport(birdDataFilePath, birdObjD, fmt=birdDataFormat)
|
|
110
|
+
logger.info("Storing %d definitions (status=%r) path: %s ", len(birdObjD), ok, birdDataFilePath)
|
|
111
|
+
#
|
|
112
|
+
endTime = time.time()
|
|
113
|
+
logger.info("Loaded/reloaded %d definitions (%.4f seconds)", len(birdObjD), endTime - startTime)
|
|
114
|
+
return birdObjD
|
|
115
|
+
|
|
116
|
+
def __fetchUrl(self, urlTarget, dirPath, useCache=False):
|
|
117
|
+
fU = FileUtil()
|
|
118
|
+
fn = fU.getFileName(urlTarget)
|
|
119
|
+
filePath = os.path.join(dirPath, fn)
|
|
120
|
+
if not (useCache and fU.exists(filePath)):
|
|
121
|
+
startTime = time.time()
|
|
122
|
+
ok2 = fU.get(urlTarget, filePath)
|
|
123
|
+
endTime = time.time()
|
|
124
|
+
if ok2:
|
|
125
|
+
logger.info("Fetched %s for resource file %s (status = %r) (%.4f seconds)", urlTarget, filePath, ok2, endTime - startTime)
|
|
126
|
+
else:
|
|
127
|
+
logger.error("Failing fetch for %s for resource file %s (status = %r) (%.4f seconds)", urlTarget, filePath, ok2, endTime - startTime)
|
|
128
|
+
#
|
|
129
|
+
return filePath
|
|
130
|
+
|
|
131
|
+
def __readBirdDefinitions(self, birdFilePath, molLimit=None):
|
|
132
|
+
birdD = {}
|
|
133
|
+
try:
|
|
134
|
+
startTime = time.time()
|
|
135
|
+
logger.info("Reading definitions from %s", birdFilePath)
|
|
136
|
+
dataContainerL = self.__mU.doImport(birdFilePath, fmt="mmcif")
|
|
137
|
+
endTime = time.time()
|
|
138
|
+
logger.info("Read %s with %d BIRD definitions (%.4f seconds)", birdFilePath, len(dataContainerL), endTime - startTime)
|
|
139
|
+
# -------
|
|
140
|
+
startTime = time.time()
|
|
141
|
+
dataContainerL = dataContainerL[:molLimit] if molLimit else dataContainerL
|
|
142
|
+
for dataContainer in dataContainerL:
|
|
143
|
+
prdId = prdReleaseStatus = representAs = chemCompId = classType = None
|
|
144
|
+
if dataContainer.exists("pdbx_reference_molecule"):
|
|
145
|
+
prdObj = dataContainer.getObj("pdbx_reference_molecule")
|
|
146
|
+
prdId = prdObj.getValueOrDefault("prd_id", 0, defaultValue=None)
|
|
147
|
+
prdReleaseStatus = prdObj.getValueOrDefault("release_status", 0, defaultValue=None)
|
|
148
|
+
representAs = prdObj.getValueOrDefault("represent_as", 0, defaultValue=None)
|
|
149
|
+
chemCompId = prdObj.getValueOrDefault("chem_comp_id", 0, defaultValue=None)
|
|
150
|
+
classType = prdObj.getValueOrDefault("class", 0, defaultValue=None)
|
|
151
|
+
if prdId:
|
|
152
|
+
birdD[prdId] = {"releaseStatus": prdReleaseStatus, "representAs": representAs, "chemCompId": chemCompId, "class": classType}
|
|
153
|
+
endTime = time.time()
|
|
154
|
+
logger.info("Processed %d definitions (%.4f seconds)", len(birdD), endTime - startTime)
|
|
155
|
+
except Exception as e:
|
|
156
|
+
logger.exception("Failing with %s", str(e))
|
|
157
|
+
#
|
|
158
|
+
return birdD
|
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
##
|
|
2
|
+
# File: CARDProvider.py
|
|
3
|
+
# Date: 28-Nov-2020 jdw
|
|
4
|
+
#
|
|
5
|
+
# Updated:
|
|
6
|
+
#
|
|
7
|
+
##
|
|
8
|
+
"""
|
|
9
|
+
Accessors for CARD molecule data.
|
|
10
|
+
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
import logging
|
|
14
|
+
import os.path
|
|
15
|
+
import time
|
|
16
|
+
|
|
17
|
+
import networkx
|
|
18
|
+
import obonet
|
|
19
|
+
|
|
20
|
+
from rcsb.utils.io.FileUtil import FileUtil
|
|
21
|
+
from rcsb.utils.io.MarshalUtil import MarshalUtil
|
|
22
|
+
|
|
23
|
+
logger = logging.getLogger(__name__)
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
class CARDProvider:
|
|
27
|
+
"""Accessors for CARD molecule data."""
|
|
28
|
+
|
|
29
|
+
def __init__(self, **kwargs):
|
|
30
|
+
#
|
|
31
|
+
self.__cachePath = kwargs.get("cachePath", ".")
|
|
32
|
+
self.__dirPath = os.path.join(self.__cachePath, "CARD-molecules")
|
|
33
|
+
#
|
|
34
|
+
self.__mU = MarshalUtil(workPath=self.__dirPath)
|
|
35
|
+
self.__oD, self.__version = self.__reload(self.__dirPath, **kwargs)
|
|
36
|
+
#
|
|
37
|
+
|
|
38
|
+
def testCache(self, minCount=350):
|
|
39
|
+
if self.__oD and len(self.__oD) >= minCount:
|
|
40
|
+
logger.info("CARD mapping length (%d)", len(self.__oD))
|
|
41
|
+
return True
|
|
42
|
+
else:
|
|
43
|
+
return False
|
|
44
|
+
|
|
45
|
+
def __reload(self, dirPath, **kwargs):
|
|
46
|
+
oD = None
|
|
47
|
+
startTime = time.time()
|
|
48
|
+
useCache = kwargs.get("useCache", True)
|
|
49
|
+
#
|
|
50
|
+
cardDumpUrl = kwargs.get("CARDDumpUrl", "https://card.mcmaster.ca/latest/ontology")
|
|
51
|
+
ok = False
|
|
52
|
+
fU = FileUtil()
|
|
53
|
+
cardDumpFileName = "card-ontology.tar.bz2"
|
|
54
|
+
cardDumpPath = os.path.join(dirPath, cardDumpFileName)
|
|
55
|
+
cardDumpDirPath = os.path.join(dirPath, "dump")
|
|
56
|
+
#
|
|
57
|
+
fU.mkdir(dirPath)
|
|
58
|
+
cardDataPath = os.path.join(dirPath, "card-molecules.json")
|
|
59
|
+
#
|
|
60
|
+
logger.info("useCache %r CARDDumpPath %r", useCache, cardDumpPath)
|
|
61
|
+
if useCache and self.__mU.exists(cardDataPath):
|
|
62
|
+
oD = self.__mU.doImport(cardDataPath, fmt="json")
|
|
63
|
+
else:
|
|
64
|
+
logger.info("Fetching url %s path %s", cardDumpUrl, cardDumpPath)
|
|
65
|
+
ok = fU.get(cardDumpUrl, cardDumpPath)
|
|
66
|
+
fU.mkdir(cardDumpDirPath)
|
|
67
|
+
fU.uncompress(cardDumpPath, outputDir=cardDumpDirPath)
|
|
68
|
+
fU.unbundleTarfile(os.path.join(cardDumpDirPath, cardDumpFileName[:-4]), dirPath=cardDumpDirPath)
|
|
69
|
+
logger.info("Completed fetch (%r) at %s (%.4f seconds)", ok, time.strftime("%Y %m %d %H:%M:%S", time.localtime()), time.time() - startTime)
|
|
70
|
+
#
|
|
71
|
+
oD = self.__parseCardData(os.path.join(cardDumpDirPath, "aro.obo"))
|
|
72
|
+
ok = self.__mU.doExport(cardDataPath, oD, fmt="json", indent=3)
|
|
73
|
+
# ---
|
|
74
|
+
return oD
|
|
75
|
+
|
|
76
|
+
def __parseCardData(self, filePath):
|
|
77
|
+
"""Parse CARD ontology file describing antibiotic molecules.
|
|
78
|
+
|
|
79
|
+
Args:
|
|
80
|
+
filePath (str): output file path for CARD molecule data
|
|
81
|
+
|
|
82
|
+
Returns:
|
|
83
|
+
(dict, version): dictionary of select CARD data, version string
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
Note: https://card.mcmaster.ca/aro/0000003
|
|
87
|
+
"""
|
|
88
|
+
oD = {}
|
|
89
|
+
version = None
|
|
90
|
+
try:
|
|
91
|
+
cardGraph = obonet.read_obo(filePath)
|
|
92
|
+
logger.info("CARD graph nodes (%d) edges (%d) directed %r", len(cardGraph), cardGraph.number_of_edges(), networkx.is_directed_acyclic_graph(cardGraph))
|
|
93
|
+
for ky, gD in cardGraph.nodes(data=True):
|
|
94
|
+
# logger.info("ky %r gD %r", ky, gD.keys())
|
|
95
|
+
if "xref" in gD:
|
|
96
|
+
# oD["name"] = gD["name"]
|
|
97
|
+
# oD["pubChemCId"] = gD["xref"]
|
|
98
|
+
oD[ky] = {"name": gD["name"], "pubChemCId": gD["xref"], "description": gD["def"]}
|
|
99
|
+
logger.info("Parsed CARD molecule references (%d)", len(oD))
|
|
100
|
+
except Exception as e:
|
|
101
|
+
logger.exception("Failing using %r with %s", filePath, str(e))
|
|
102
|
+
return oD, version
|
|
@@ -0,0 +1,132 @@
|
|
|
1
|
+
##
|
|
2
|
+
# File: CODProvider.py
|
|
3
|
+
# Date: 8-Feb-2021 jdw
|
|
4
|
+
#
|
|
5
|
+
# Updated:
|
|
6
|
+
#
|
|
7
|
+
##
|
|
8
|
+
"""
|
|
9
|
+
Accessors for Crystallographic Open Database (COD) molecule data.
|
|
10
|
+
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
import logging
|
|
14
|
+
import os.path
|
|
15
|
+
import time
|
|
16
|
+
|
|
17
|
+
from rcsb.utils.io.FileUtil import FileUtil
|
|
18
|
+
from rcsb.utils.io.MarshalUtil import MarshalUtil
|
|
19
|
+
|
|
20
|
+
logger = logging.getLogger(__name__)
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
class CODProvider:
|
|
24
|
+
"""Accessors for COD molecule data."""
|
|
25
|
+
|
|
26
|
+
def __init__(self, **kwargs):
|
|
27
|
+
#
|
|
28
|
+
self.__cachePath = kwargs.get("cachePath", ".")
|
|
29
|
+
self.__dirPath = os.path.join(self.__cachePath, "COD-molecules")
|
|
30
|
+
self.__codCifDirPath = os.path.join(self.__dirPath, "CIF")
|
|
31
|
+
self.__useCache = kwargs.get("useCache", True)
|
|
32
|
+
codSmilesDumpUrl = kwargs.get("CODSmilesDumpUrl", "http://www.crystallography.net/cod/smi/allcod.smi")
|
|
33
|
+
#
|
|
34
|
+
self.__codCifTemplateUrl = "http://www.crystallography.net/cod/"
|
|
35
|
+
self.__mU = MarshalUtil(workPath=self.__dirPath)
|
|
36
|
+
ok = self.__reload(self.__dirPath, codSmilesDumpUrl, self.__useCache)
|
|
37
|
+
#
|
|
38
|
+
logger.info("COD SMILES data status (%r)", ok)
|
|
39
|
+
#
|
|
40
|
+
|
|
41
|
+
def testCache(self, minCount=200000):
|
|
42
|
+
_ = minCount
|
|
43
|
+
return True
|
|
44
|
+
|
|
45
|
+
def getSmilesPath(self):
|
|
46
|
+
return os.path.join(self.__dirPath, "cod-molecules.smi")
|
|
47
|
+
|
|
48
|
+
def __reload(self, dirPath, codSmilesDumpUrl, useCache):
|
|
49
|
+
startTime = time.time()
|
|
50
|
+
|
|
51
|
+
ok = False
|
|
52
|
+
fU = FileUtil()
|
|
53
|
+
codSmilesDumpFileName = "allcod.smi"
|
|
54
|
+
codSmilesDumpPath = os.path.join(dirPath, codSmilesDumpFileName)
|
|
55
|
+
#
|
|
56
|
+
fU.mkdir(dirPath)
|
|
57
|
+
codSmilesDataPath = self.getSmilesPath()
|
|
58
|
+
#
|
|
59
|
+
logger.info("useCache %r CODSmilesDumpPath %r", useCache, codSmilesDumpPath)
|
|
60
|
+
if useCache and fU.exists(codSmilesDataPath):
|
|
61
|
+
#
|
|
62
|
+
ok = True
|
|
63
|
+
else:
|
|
64
|
+
logger.info("Fetching url %s path %s", codSmilesDumpUrl, codSmilesDumpPath)
|
|
65
|
+
ok = fU.get(codSmilesDumpUrl, codSmilesDumpPath)
|
|
66
|
+
logger.info("Completed fetch (%r) at %s (%.4f seconds)", ok, time.strftime("%Y %m %d %H:%M:%S", time.localtime()), time.time() - startTime)
|
|
67
|
+
#
|
|
68
|
+
numSmiles = self.__reformatCodSmilesData(codSmilesDumpPath, codSmilesDataPath)
|
|
69
|
+
ok = numSmiles > 200000
|
|
70
|
+
# ---
|
|
71
|
+
return ok
|
|
72
|
+
|
|
73
|
+
def __reformatCodSmilesData(self, inpFilePath, outFilePath):
|
|
74
|
+
"""Reformat COD SMILES data -
|
|
75
|
+
|
|
76
|
+
Args:
|
|
77
|
+
inpFilePath (str): input file path for COD molecule data
|
|
78
|
+
outFilePath (str): reformatted file path for COD molecule data
|
|
79
|
+
|
|
80
|
+
Returns:
|
|
81
|
+
(int): number of reformatted SMILES records
|
|
82
|
+
"""
|
|
83
|
+
sCount = 0
|
|
84
|
+
try:
|
|
85
|
+
nl = "\n"
|
|
86
|
+
with open(inpFilePath, "r", encoding="utf-8") as ifh, open(outFilePath, "w", encoding="utf-8") as ofh:
|
|
87
|
+
for line in ifh.readlines():
|
|
88
|
+
ff = line[:-1].split("\t")
|
|
89
|
+
ff.reverse()
|
|
90
|
+
ofh.write("\t".join(ff) + nl)
|
|
91
|
+
sCount += 1
|
|
92
|
+
#
|
|
93
|
+
logger.info("Parsed COD molecule references (%d)", sCount)
|
|
94
|
+
except Exception as e:
|
|
95
|
+
logger.exception("Failing using %r with %s", inpFilePath, str(e))
|
|
96
|
+
return sCount
|
|
97
|
+
|
|
98
|
+
def getCodSmilesList(self):
|
|
99
|
+
smiTupL = []
|
|
100
|
+
inpFilePath = self.getSmilesPath()
|
|
101
|
+
try:
|
|
102
|
+
with open(inpFilePath, "r", encoding="utf-8") as ifh:
|
|
103
|
+
for line in ifh.readlines():
|
|
104
|
+
ff = line[:-1].split("\t")
|
|
105
|
+
smiTupL.append((ff[0], ff[1]))
|
|
106
|
+
return smiTupL
|
|
107
|
+
except Exception as e:
|
|
108
|
+
logger.exception("Failing for %r with %s", inpFilePath, str(e))
|
|
109
|
+
|
|
110
|
+
def getCifPath(self, codId):
|
|
111
|
+
return os.path.join(self.__codCifDirPath, codId[-1], codId + ".cif")
|
|
112
|
+
|
|
113
|
+
def fetchCif(self, codId):
|
|
114
|
+
ok = False
|
|
115
|
+
try:
|
|
116
|
+
startTime = time.time()
|
|
117
|
+
fU = FileUtil()
|
|
118
|
+
fU.mkdir(self.__codCifDirPath)
|
|
119
|
+
codCifPath = self.getCifPath(codId)
|
|
120
|
+
#
|
|
121
|
+
logger.info("useCache %r codCifPath %r", self.__useCache, codCifPath)
|
|
122
|
+
if self.__useCache and fU.exists(codCifPath):
|
|
123
|
+
ok = True
|
|
124
|
+
else:
|
|
125
|
+
codCifUrl = os.path.join(self.__codCifTemplateUrl, codId + ".cif")
|
|
126
|
+
logger.info("Fetching url %s path %s", codCifUrl, codCifPath)
|
|
127
|
+
ok = fU.get(codCifUrl, codCifPath)
|
|
128
|
+
logger.info("Completed fetch (%r) at %s (%.4f seconds)", ok, time.strftime("%Y %m %d %H:%M:%S", time.localtime()), time.time() - startTime)
|
|
129
|
+
except Exception as e:
|
|
130
|
+
logger.exception("Failing for %r with %s", codId, str(e))
|
|
131
|
+
# ---
|
|
132
|
+
return ok
|