rcsb.exdb 1.31__py3-none-any.whl → 1.33__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- rcsb/exdb/tree/TreeNodeListWorker.py +72 -49
- {rcsb_exdb-1.31.dist-info → rcsb_exdb-1.33.dist-info}/METADATA +2 -2
- {rcsb_exdb-1.31.dist-info → rcsb_exdb-1.33.dist-info}/RECORD +5 -42
- rcsb/exdb/tests/TEST-EXDB-CLI-EXEC.sh +0 -19
- rcsb/exdb/tests/TEST-EXDB-CLI-REFSEQ-EXEC.sh +0 -12
- rcsb/exdb/tests/__init__.py +0 -0
- rcsb/exdb/tests/fixtureDictMethodResourceProvider.py +0 -104
- rcsb/exdb/tests/fixturePdbxLoader.py +0 -298
- rcsb/exdb/tests/test-data/components-abbrev.cif +0 -2739
- rcsb/exdb/tests/test-data/prdcc-abbrev.cif +0 -9171
- rcsb/exdb/tests/testAnnotationExtractor.py +0 -79
- rcsb/exdb/tests/testBranchedEntityExtractor.py +0 -81
- rcsb/exdb/tests/testChemRefLoader.py +0 -106
- rcsb/exdb/tests/testChemRefMappingProvider.py +0 -95
- rcsb/exdb/tests/testCitationAdapter.py +0 -97
- rcsb/exdb/tests/testCitationExtractor.py +0 -93
- rcsb/exdb/tests/testCitationUtils.py +0 -92
- rcsb/exdb/tests/testEntryInfoEtlWorkflow.py +0 -70
- rcsb/exdb/tests/testEntryInfoProvider.py +0 -97
- rcsb/exdb/tests/testGlycanEtlWorkflow.py +0 -70
- rcsb/exdb/tests/testGlycanProvider.py +0 -98
- rcsb/exdb/tests/testGlycanUtils.py +0 -64
- rcsb/exdb/tests/testLigandNeighborMappingProvider.py +0 -90
- rcsb/exdb/tests/testObjectExtractor.py +0 -342
- rcsb/exdb/tests/testObjectTransformer.py +0 -83
- rcsb/exdb/tests/testObjectUpdater.py +0 -120
- rcsb/exdb/tests/testPolymerEntityExtractor.py +0 -93
- rcsb/exdb/tests/testPubChemDataCacheProvider.py +0 -124
- rcsb/exdb/tests/testPubChemEtlWorkflow.py +0 -134
- rcsb/exdb/tests/testPubChemEtlWrapper.py +0 -155
- rcsb/exdb/tests/testPubChemIndexCacheProvider.py +0 -123
- rcsb/exdb/tests/testReferenceSequenceAnnotationAdapter.py +0 -106
- rcsb/exdb/tests/testReferenceSequenceAssignmentAdapter.py +0 -121
- rcsb/exdb/tests/testReferenceSequenceAssignmentAdapterValidate.py +0 -122
- rcsb/exdb/tests/testReferenceSequenceAssignmentProvider.py +0 -117
- rcsb/exdb/tests/testReferenceSequenceCacheProvider.py +0 -94
- rcsb/exdb/tests/testTaxonomyExtractor.py +0 -75
- rcsb/exdb/tests/testTreeNodeListWorker.py +0 -111
- rcsb/exdb/tests/testUniProtCoreEtlWorker.py +0 -99
- rcsb/exdb/tests/testUniProtExtractor.py +0 -77
- {rcsb_exdb-1.31.dist-info → rcsb_exdb-1.33.dist-info}/WHEEL +0 -0
- {rcsb_exdb-1.31.dist-info → rcsb_exdb-1.33.dist-info}/licenses/LICENSE +0 -0
|
@@ -12,6 +12,8 @@
|
|
|
12
12
|
# 23-Jan-2025 dwp Change indexed field from 'update_id' to 'id'
|
|
13
13
|
# 7-Aug-2025 dwp Change target DB and collection names to "dw" and "tree_*" (via configuration file);
|
|
14
14
|
# Make use of configuration file for loading tree node lists and setting indexed fields
|
|
15
|
+
# 6-Jan-2026 dwp Raise error if a tree node list is empty or fails to load;
|
|
16
|
+
# Add support for providing a manual list of tree node lists to load (for testing)
|
|
15
17
|
#
|
|
16
18
|
##
|
|
17
19
|
__docformat__ = "google en"
|
|
@@ -41,7 +43,20 @@ logger = logging.getLogger(__name__)
|
|
|
41
43
|
class TreeNodeListWorker(object):
|
|
42
44
|
"""Prepare and load repository holdings and repository update data."""
|
|
43
45
|
|
|
44
|
-
def __init__(
|
|
46
|
+
def __init__(
|
|
47
|
+
self,
|
|
48
|
+
cfgOb,
|
|
49
|
+
cachePath,
|
|
50
|
+
numProc=1,
|
|
51
|
+
chunkSize=10,
|
|
52
|
+
maxStepLength=4000,
|
|
53
|
+
readBackCheck=False,
|
|
54
|
+
documentLimit=None,
|
|
55
|
+
verbose=False,
|
|
56
|
+
useCache=False,
|
|
57
|
+
useFilteredLists=False,
|
|
58
|
+
treeCollectionList=None,
|
|
59
|
+
):
|
|
45
60
|
self.__cfgOb = cfgOb
|
|
46
61
|
self.__cachePath = os.path.abspath(cachePath)
|
|
47
62
|
self.__readBackCheck = readBackCheck
|
|
@@ -55,6 +70,7 @@ class TreeNodeListWorker(object):
|
|
|
55
70
|
self.__statusList = []
|
|
56
71
|
self.__useCache = useCache
|
|
57
72
|
self.__useFilteredLists = useFilteredLists
|
|
73
|
+
self.__treeCollectionList = treeCollectionList if treeCollectionList else [] # Manually specify tree collections to load
|
|
58
74
|
|
|
59
75
|
def __updateStatus(self, updateId, databaseName, collectionName, status, startTimestamp):
|
|
60
76
|
try:
|
|
@@ -95,43 +111,49 @@ class TreeNodeListWorker(object):
|
|
|
95
111
|
ATTRIBUTE_NAMES:
|
|
96
112
|
- parents
|
|
97
113
|
"""
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
114
|
+
ok = True
|
|
115
|
+
useCache = self.__useCache
|
|
116
|
+
#
|
|
117
|
+
logger.info("Starting with cache path %r (useCache=%r)", self.__cachePath, useCache)
|
|
118
|
+
#
|
|
119
|
+
self.__statusList = []
|
|
120
|
+
desp = DataExchangeStatus()
|
|
121
|
+
statusStartTimestamp = desp.setStartTime()
|
|
122
|
+
dl = DocumentLoader(
|
|
123
|
+
self.__cfgOb,
|
|
124
|
+
self.__cachePath,
|
|
125
|
+
self.__resourceName,
|
|
126
|
+
numProc=self.__numProc,
|
|
127
|
+
chunkSize=self.__chunkSize,
|
|
128
|
+
maxStepLength=self.__maxStepLength,
|
|
129
|
+
documentLimit=self.__documentLimit,
|
|
130
|
+
verbose=self.__verbose,
|
|
131
|
+
readBackCheck=self.__readBackCheck,
|
|
132
|
+
)
|
|
133
|
+
#
|
|
134
|
+
sectionName = "tree_node_lists_configuration"
|
|
135
|
+
databaseNameMongo = self.__cfgOb.get("DATABASE_NAME", sectionName=sectionName)
|
|
136
|
+
collectionNameList = self.__cfgOb.get("COLLECTION_NAME_LIST", sectionName=sectionName)
|
|
137
|
+
collectionIndexList = self.__cfgOb.get("COLLECTION_INDICES", sectionName=sectionName)
|
|
138
|
+
# databaseNameMongo = 'dw'
|
|
139
|
+
# collectionNameList = ['tree_taxonomy', 'tree_ec', 'tree_scop', 'tree_scop2', 'tree_cath', 'tree_atc', 'tree_card', 'tree_ecod', 'tree_go']
|
|
140
|
+
# collectionIndexList = [{'INDEX_NAME': 'primary', 'ATTRIBUTE_NAMES': ['id']}, {'INDEX_NAME': 'index_2', 'ATTRIBUTE_NAMES': ['parents']}]
|
|
141
|
+
|
|
142
|
+
if len(self.__treeCollectionList) > 0:
|
|
143
|
+
collectionNameList = [col for col in self.__treeCollectionList]
|
|
144
|
+
|
|
145
|
+
# collectionVersion = self.__cfgOb.get("COLLECTION_VERSION_STRING", sectionName=sectionName)
|
|
146
|
+
# addValues = {"_schema_version": collectionVersion}
|
|
147
|
+
addValues = None
|
|
148
|
+
|
|
149
|
+
for collectionName in collectionNameList:
|
|
150
|
+
nL = self.__getTreeDocList(collectionName, useCache)
|
|
151
|
+
if doLoad:
|
|
152
|
+
if not nL or len(nL) == 0:
|
|
153
|
+
logger.error("Empty node list returned for collectionName %r", collectionName)
|
|
154
|
+
ok = False
|
|
155
|
+
else:
|
|
156
|
+
okL = dl.load(
|
|
135
157
|
databaseNameMongo,
|
|
136
158
|
collectionName,
|
|
137
159
|
loadType=loadType,
|
|
@@ -140,18 +162,19 @@ class TreeNodeListWorker(object):
|
|
|
140
162
|
addValues=addValues,
|
|
141
163
|
schemaLevel=None,
|
|
142
164
|
indexDL=collectionIndexList
|
|
143
|
-
)
|
|
165
|
+
)
|
|
166
|
+
ok = okL and ok
|
|
144
167
|
self.__updateStatus(updateId, databaseNameMongo, collectionName, ok, statusStartTimestamp)
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
return
|
|
168
|
+
|
|
169
|
+
logger.info(
|
|
170
|
+
"Completed load of tree node list for database %r, collection %r, len(nL) %r (status %r)",
|
|
171
|
+
databaseNameMongo, collectionName, len(nL), okL
|
|
172
|
+
)
|
|
173
|
+
# ---
|
|
174
|
+
logger.info("Completed tree node list loading operations with loadType %r (status %r)", loadType, ok)
|
|
175
|
+
if not ok:
|
|
176
|
+
raise ValueError("Failed to load at least one tree node list.")
|
|
177
|
+
return ok
|
|
155
178
|
|
|
156
179
|
def __checkTaxonNodeList(self, nL):
|
|
157
180
|
eCount = 0
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: rcsb.exdb
|
|
3
|
-
Version: 1.
|
|
3
|
+
Version: 1.33
|
|
4
4
|
Summary: RCSB Python ExDB data extraction and loading workflows
|
|
5
5
|
Project-URL: Homepage, https://github.com/rcsb/py-rcsb_exdb
|
|
6
6
|
Author-email: John Westbrook <john.westbrook@rcsb.org>
|
|
@@ -26,7 +26,7 @@ Requires-Dist: rcsb-utils-ec>=0.25
|
|
|
26
26
|
Requires-Dist: rcsb-utils-go>=0.18
|
|
27
27
|
Requires-Dist: rcsb-utils-io>=1.48
|
|
28
28
|
Requires-Dist: rcsb-utils-seq>=0.82
|
|
29
|
-
Requires-Dist: rcsb-utils-struct>=0.
|
|
29
|
+
Requires-Dist: rcsb-utils-struct>=0.51
|
|
30
30
|
Requires-Dist: rcsb-utils-targets>=0.82
|
|
31
31
|
Requires-Dist: rcsb-utils-taxonomy>=0.43
|
|
32
32
|
Provides-Extra: tests
|
|
@@ -43,44 +43,7 @@ rcsb/exdb/seq/TaxonomyExtractor.py,sha256=I7jsb5Kanrnh4X-znl9kZPZMJ7o2dp4fsnp2IW
|
|
|
43
43
|
rcsb/exdb/seq/UniProtCoreEtlWorker.py,sha256=-fEojXF3lAJ1tbMsPIxT9In6ooiPThuKSoIRQ0YlZ1s,7590
|
|
44
44
|
rcsb/exdb/seq/UniProtExtractor.py,sha256=pR_A9e82YvbQ813M8rNPu1bCPOHMjGnCqJmLDMM23Qo,2695
|
|
45
45
|
rcsb/exdb/seq/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
46
|
-
rcsb/exdb/
|
|
47
|
-
rcsb/exdb/tests/TEST-EXDB-CLI-REFSEQ-EXEC.sh,sha256=CPQ8rRpfBVIl5r6IFt2t-P4aBBIWUo4NioR7M0LM6Nc,769
|
|
48
|
-
rcsb/exdb/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
49
|
-
rcsb/exdb/tests/fixtureDictMethodResourceProvider.py,sha256=h_iRfmZR9UG0nFYnVoPDtAE3VPpyrnwJRXbrdfeh0gY,3999
|
|
50
|
-
rcsb/exdb/tests/fixturePdbxLoader.py,sha256=wfXyJ4k2Z-85J6sCSArf-eAuYQVbsTilwWmnm8cNPa4,10167
|
|
51
|
-
rcsb/exdb/tests/testAnnotationExtractor.py,sha256=rpeB4RtOcYD27S3ZwnePHHSaac2j4bmvqfu-wJ5254U,2657
|
|
52
|
-
rcsb/exdb/tests/testBranchedEntityExtractor.py,sha256=efchTzuQylD7L7Kc0j-rEvBeLjnPzvSWVLA8I7JIBys,2936
|
|
53
|
-
rcsb/exdb/tests/testChemRefLoader.py,sha256=pZqWiSIQZ9bfZMJplw_ym9swCe0N6rjq63mfcqWtNO0,3696
|
|
54
|
-
rcsb/exdb/tests/testChemRefMappingProvider.py,sha256=wxypTg-Y5DsAksT4a2x2yh1_JCf-7HqP9z0jKDmNcNg,3353
|
|
55
|
-
rcsb/exdb/tests/testCitationAdapter.py,sha256=ZlvG6aTHzKVnREj3LXQd1iniwC-KXR45vbyzBgdoiu8,3625
|
|
56
|
-
rcsb/exdb/tests/testCitationExtractor.py,sha256=7Xg75JHtnZkfrFnZVNggLRVFeRNW6V5O0tGse40C9fc,3141
|
|
57
|
-
rcsb/exdb/tests/testCitationUtils.py,sha256=lCn5n_49vAnQIaUd4fSY-sQ6Ld5r5G32WRaprErryo8,3064
|
|
58
|
-
rcsb/exdb/tests/testEntryInfoEtlWorkflow.py,sha256=TutfC949U0nDkFbmMdXo_MgbxMfvll4O-5C7N8Qj3TA,2340
|
|
59
|
-
rcsb/exdb/tests/testEntryInfoProvider.py,sha256=2Ti-o2A8QlB0reJHFqXHjyrvehlLRFtycFhCTBTY_d0,3360
|
|
60
|
-
rcsb/exdb/tests/testGlycanEtlWorkflow.py,sha256=Olml-71jiWAGOU-NQNHiRSPrew7RJtLK3R6u6WTlUSc,2295
|
|
61
|
-
rcsb/exdb/tests/testGlycanProvider.py,sha256=n15qTncV289o-E5J_AsizPW1tgeCHZH8A7_qXG2Afr4,3201
|
|
62
|
-
rcsb/exdb/tests/testGlycanUtils.py,sha256=h59g_iuEEzhcWRW50VdYlRHw3UrpqiRYgJZKDU4C9Wc,1953
|
|
63
|
-
rcsb/exdb/tests/testLigandNeighborMappingProvider.py,sha256=JjY4y5a4ZuZPJ4gdOxIwRLVz5U6F0f0K963OeDZmFAM,3144
|
|
64
|
-
rcsb/exdb/tests/testObjectExtractor.py,sha256=aHP6q9UQqDFd44u8KNQrnfTWy8Rf9dQ8FnHhUHAmqq0,14122
|
|
65
|
-
rcsb/exdb/tests/testObjectTransformer.py,sha256=7fPWTfN4G6wFj6MZkw_L0dtONMsVBlBCzoHvxzrgsUo,2944
|
|
66
|
-
rcsb/exdb/tests/testObjectUpdater.py,sha256=Kq-Mk4ZbD-eMoIsR3zGJvamFq-OEX4tFXyPB9FePWxc,4950
|
|
67
|
-
rcsb/exdb/tests/testPolymerEntityExtractor.py,sha256=RTsEWZDarrtdVn2Xy7fbPIrZTw7Nd9VJ21aBBr3QFU0,3536
|
|
68
|
-
rcsb/exdb/tests/testPubChemDataCacheProvider.py,sha256=K2Fg4Bibtdg9Z9zLwwJqZDa-XlIKHiSb9A6ewIa2-sM,4679
|
|
69
|
-
rcsb/exdb/tests/testPubChemEtlWorkflow.py,sha256=vENOnc7D-EukdjElvhsZqTL0h5N6alPso4Gn3rDNMXg,5187
|
|
70
|
-
rcsb/exdb/tests/testPubChemEtlWrapper.py,sha256=yGg0HvtYvFTvEEK9OsqvSSRVzCIG90UPEK_Lh7SifAk,6511
|
|
71
|
-
rcsb/exdb/tests/testPubChemIndexCacheProvider.py,sha256=KVPfgnMdp7V57LnuwLLkbcZ-DjGHdEx4q0V1RAyOCXw,4947
|
|
72
|
-
rcsb/exdb/tests/testReferenceSequenceAnnotationAdapter.py,sha256=hABmh743REIhFe3vP_dKr0uUvUH6-Ci71fpHGUIzt6E,4495
|
|
73
|
-
rcsb/exdb/tests/testReferenceSequenceAssignmentAdapter.py,sha256=g9-t6-TMZNqxtjgZy9cYCXA19ZkOfjb0un7o8sasuAA,4897
|
|
74
|
-
rcsb/exdb/tests/testReferenceSequenceAssignmentAdapterValidate.py,sha256=p3wPa-9_Il9eHP3fQGI1ag2FE0mvuOdeGu2uet6APOU,5005
|
|
75
|
-
rcsb/exdb/tests/testReferenceSequenceAssignmentProvider.py,sha256=6qzk-xt0QVrX4IC8Ml6HnDzFcee14vIUs14AKAWFg9w,4631
|
|
76
|
-
rcsb/exdb/tests/testReferenceSequenceCacheProvider.py,sha256=thQnpv1dxnL46PfZS6alhyS2N_rnATRmvh8SU9HJHAM,3469
|
|
77
|
-
rcsb/exdb/tests/testTaxonomyExtractor.py,sha256=NZv0UFBnnRUxQN3gzPIruOqyJEG6F81avI8tQu9ILao,2536
|
|
78
|
-
rcsb/exdb/tests/testTreeNodeListWorker.py,sha256=PiVaSOG0140fonhHsp0yxft3KlYaBzDb2hwmPKZxy5Y,3653
|
|
79
|
-
rcsb/exdb/tests/testUniProtCoreEtlWorker.py,sha256=8LyJjeMiKkGSYZ1641xpO1clKSJSESWrHywrNqMeJ9o,3403
|
|
80
|
-
rcsb/exdb/tests/testUniProtExtractor.py,sha256=jNNAI1GIedEbZhpdG1C9CjuZskUwxFg8jxgZJi1tr2U,2604
|
|
81
|
-
rcsb/exdb/tests/test-data/components-abbrev.cif,sha256=Vodm2R_8ipO2X4e8W3Wa1MiH3YF7RSczdtXC0TA13sM,139314
|
|
82
|
-
rcsb/exdb/tests/test-data/prdcc-abbrev.cif,sha256=pA2yCImadbuh51q59fdIOsGnyXVBPFv-8Z8Zrk2jUqA,415606
|
|
83
|
-
rcsb/exdb/tree/TreeNodeListWorker.py,sha256=VLd7MWCxw9fONoC3xYbjvARp2O2V8Vyy-kUZnwQWi30,10233
|
|
46
|
+
rcsb/exdb/tree/TreeNodeListWorker.py,sha256=jaCe21zVfj5h5g_n5ibwG33JoaX0nBqXCSLiw8DSF0A,10903
|
|
84
47
|
rcsb/exdb/tree/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
85
48
|
rcsb/exdb/utils/ObjectAdapterBase.py,sha256=w-MGvs-TFQXzfgOfAX3aNyCfaN9gY8WP-7MU2FcMAYs,466
|
|
86
49
|
rcsb/exdb/utils/ObjectExtractor.py,sha256=fAJ-WW_-80h_s_XSDdZYi1I2pltb-uQ3teOtCkcyznk,11057
|
|
@@ -92,7 +55,7 @@ rcsb/exdb/wf/EntryInfoEtlWorkflow.py,sha256=YVr75Wz1BPjLr_satd28B9BeD3QL6HwmkR17
|
|
|
92
55
|
rcsb/exdb/wf/GlycanEtlWorkflow.py,sha256=oJ6wf438K2e-eLmy8Ni3MCPxjAKgVJY38SWO885gnmg,2820
|
|
93
56
|
rcsb/exdb/wf/PubChemEtlWorkflow.py,sha256=fNX3A6kf0S1XiJMz7ywNpFuuua5lT3XaUFjcCJtvQsU,11235
|
|
94
57
|
rcsb/exdb/wf/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
95
|
-
rcsb_exdb-1.
|
|
96
|
-
rcsb_exdb-1.
|
|
97
|
-
rcsb_exdb-1.
|
|
98
|
-
rcsb_exdb-1.
|
|
58
|
+
rcsb_exdb-1.33.dist-info/METADATA,sha256=0_XboU3Pyut6e2JCDds5wE4pPVFJHGoCMnlvOtZckME,3845
|
|
59
|
+
rcsb_exdb-1.33.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
|
|
60
|
+
rcsb_exdb-1.33.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
|
|
61
|
+
rcsb_exdb-1.33.dist-info/RECORD,,
|
|
@@ -1,19 +0,0 @@
|
|
|
1
|
-
#!/bin/bash
|
|
2
|
-
# File: TEST-EXDB-CL-EXEC.sh
|
|
3
|
-
# Date: 3-Sep-2019 jdw
|
|
4
|
-
#
|
|
5
|
-
# Examples
|
|
6
|
-
#
|
|
7
|
-
# tree node list load
|
|
8
|
-
#
|
|
9
|
-
exdb_exec_cli --mock --full --etl_tree_node_lists --rebuild_cache --cache_path ../../../CACHE --config_path ../../mock-data/config/dbload-setup-example.yml --config_name site_info_configuration >& ./test-output/LOGTREENODELIST
|
|
10
|
-
#
|
|
11
|
-
# Chemref load
|
|
12
|
-
#
|
|
13
|
-
exdb_exec_cli --mock --full --etl_chemref --cache_path ../../../CACHE --config_path ../../mock-data/config/dbload-setup-example.yml --config_name site_info_configuration >& ./test-output/LOGCHEMREF
|
|
14
|
-
#
|
|
15
|
-
# Reference sequence update
|
|
16
|
-
#
|
|
17
|
-
exdb_exec_cli --mock --upd_ref_seq --cache_path ../../../CACHE --config_path ../../mock-data/config/dbload-setup-example.yml --config_name site_info_configuration >& ./test-output/LOGUPDREFSEQ
|
|
18
|
-
#
|
|
19
|
-
#
|
|
@@ -1,12 +0,0 @@
|
|
|
1
|
-
#!/bin/bash
|
|
2
|
-
# File: TEST-EXDB-CLI-REFSEQ-EXEC.sh
|
|
3
|
-
# Date: 17-Oct-2019 jdw
|
|
4
|
-
#
|
|
5
|
-
# Reference sequence update --mock is required for example SIFTS files -
|
|
6
|
-
#
|
|
7
|
-
exdb_exec_cli --mock --upd_ref_seq --cache_path ../../../CACHE --config_path ../../mock-data/config/dbload-setup-example.yml --config_name site_info_configuration >& ./test-output/LOGUPDREFSEQ
|
|
8
|
-
#
|
|
9
|
-
exdb_exec_cli --test_req_seq_cache --mock --upd_ref_seq --cache_path ../../../CACHE --config_path ../../mock-data/config/dbload-setup-example.yml --config_name site_info_configuration >& ./test-output/LOGUPDREFSEQTEST
|
|
10
|
-
#
|
|
11
|
-
exdb_exec_cli --mock --full --etl_uniprot --cache_path ../../../CACHE --config_path ../../mock-data/config/dbload-setup-example.yml --config_name site_info_configuration >& ./test-output/LOGUNIPROT
|
|
12
|
-
#
|
rcsb/exdb/tests/__init__.py
DELETED
|
File without changes
|
|
@@ -1,104 +0,0 @@
|
|
|
1
|
-
# File: DictMethodResourceProviderFixture.py
|
|
2
|
-
# Author: J. Westbrook
|
|
3
|
-
# Date: 12-Aug-2019
|
|
4
|
-
# Version: 0.001
|
|
5
|
-
#
|
|
6
|
-
# Update:
|
|
7
|
-
|
|
8
|
-
##
|
|
9
|
-
"""
|
|
10
|
-
Fixture for setting up cached resources for dictionary method helpers
|
|
11
|
-
|
|
12
|
-
"""
|
|
13
|
-
|
|
14
|
-
__docformat__ = "google en"
|
|
15
|
-
__author__ = "John Westbrook"
|
|
16
|
-
__email__ = "jwest@rcsb.rutgers.edu"
|
|
17
|
-
__license__ = "Apache 2.0"
|
|
18
|
-
|
|
19
|
-
import logging
|
|
20
|
-
import os
|
|
21
|
-
import platform
|
|
22
|
-
import resource
|
|
23
|
-
import time
|
|
24
|
-
import unittest
|
|
25
|
-
|
|
26
|
-
from rcsb.utils.config.ConfigUtil import ConfigUtil
|
|
27
|
-
from rcsb.utils.dictionary.DictMethodResourceProvider import DictMethodResourceProvider
|
|
28
|
-
|
|
29
|
-
logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s]-%(module)s.%(funcName)s: %(message)s")
|
|
30
|
-
logger = logging.getLogger()
|
|
31
|
-
logger.setLevel(logging.INFO)
|
|
32
|
-
|
|
33
|
-
HERE = os.path.abspath(os.path.dirname(__file__))
|
|
34
|
-
TOPDIR = os.path.dirname(os.path.dirname(os.path.dirname(HERE)))
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
class DictMethodResourceProviderFixture(unittest.TestCase):
|
|
38
|
-
def setUp(self):
|
|
39
|
-
self.__cachePath = os.path.join(TOPDIR, "CACHE")
|
|
40
|
-
mockTopPath = os.path.join(TOPDIR, "rcsb", "mock-data")
|
|
41
|
-
configPath = os.path.join(TOPDIR, "rcsb", "mock-data", "config", "dbload-setup-example.yml")
|
|
42
|
-
configName = "site_info_configuration"
|
|
43
|
-
self.__configName = configName
|
|
44
|
-
self.__cfgOb = ConfigUtil(configPath=configPath, defaultSectionName=configName, mockTopPath=mockTopPath)
|
|
45
|
-
|
|
46
|
-
self.__startTime = time.time()
|
|
47
|
-
logger.debug("Starting %s at %s", self.id(), time.strftime("%Y %m %d %H:%M:%S", time.localtime()))
|
|
48
|
-
|
|
49
|
-
def tearDown(self):
|
|
50
|
-
unitS = "MB" if platform.system() == "Darwin" else "GB"
|
|
51
|
-
rusageMax = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss
|
|
52
|
-
logger.info("Maximum resident memory size %.4f %s", rusageMax / 10 ** 6, unitS)
|
|
53
|
-
endTime = time.time()
|
|
54
|
-
logger.info("Completed %s at %s (%.4f seconds)", self.id(), time.strftime("%Y %m %d %H:%M:%S", time.localtime()), endTime - self.__startTime)
|
|
55
|
-
|
|
56
|
-
def testBuildResourceCache(self):
|
|
57
|
-
"""Fixture - generate and check selected resource caches"""
|
|
58
|
-
try:
|
|
59
|
-
resourceNameL = [
|
|
60
|
-
"AtcProvider instance",
|
|
61
|
-
"DrugBankProvider instance",
|
|
62
|
-
"PubChemProvider instance",
|
|
63
|
-
"CitationReferenceProvider instance",
|
|
64
|
-
"JournalTitleAbbreviationProvider instance",
|
|
65
|
-
"EnzymeDatabaseProvider instance",
|
|
66
|
-
"PfamProvider instance",
|
|
67
|
-
"SiftsSummaryProvider instance",
|
|
68
|
-
"CathProvider instance",
|
|
69
|
-
"ScopProvider instance",
|
|
70
|
-
"EcodProvider instance",
|
|
71
|
-
"Scop2Provider instance",
|
|
72
|
-
"TaxonomyProvider instance",
|
|
73
|
-
]
|
|
74
|
-
rP = DictMethodResourceProvider(self.__cfgOb, configName=self.__configName, cachePath=self.__cachePath, restoreUseStash=False, restoreUseGit=True)
|
|
75
|
-
for resourceName in resourceNameL:
|
|
76
|
-
rP.getResource(resourceName, useCache=True, default=None, doRestore=True, doBackup=False)
|
|
77
|
-
#
|
|
78
|
-
except Exception as e:
|
|
79
|
-
logger.exception("Failing with %s", str(e))
|
|
80
|
-
self.fail()
|
|
81
|
-
|
|
82
|
-
@unittest.skip("Troubleshooting test")
|
|
83
|
-
def testRecoverResourceCache(self):
|
|
84
|
-
"""Fixture - generate and check resource caches"""
|
|
85
|
-
try:
|
|
86
|
-
rp = DictMethodResourceProvider(self.__cfgOb, configName=self.__configName, cachePath=self.__cachePath)
|
|
87
|
-
ret = rp.cacheResources(useCache=True)
|
|
88
|
-
self.assertTrue(ret)
|
|
89
|
-
except Exception as e:
|
|
90
|
-
logger.exception("Failing with %s", str(e))
|
|
91
|
-
self.fail()
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
def dictMethodResourceProviderSuite():
|
|
95
|
-
suiteSelect = unittest.TestSuite()
|
|
96
|
-
suiteSelect.addTest(DictMethodResourceProviderFixture("testBuildResourceCache"))
|
|
97
|
-
# suiteSelect.addTest(DictMethodResourceProviderFixture("testRecoverResourceCache"))
|
|
98
|
-
return suiteSelect
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
if __name__ == "__main__":
|
|
102
|
-
|
|
103
|
-
mySuite = dictMethodResourceProviderSuite()
|
|
104
|
-
unittest.TextTestRunner(verbosity=2).run(mySuite)
|