scipion-pyworkflow 3.10.5__py3-none-any.whl → 3.11.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pyworkflow/config.py +131 -67
- pyworkflow/constants.py +12 -2
- pyworkflow/object.py +3 -2
- pyworkflow/plugin.py +93 -44
- pyworkflow/project/scripts/fix_links.py +4 -1
- pyworkflow/resources/showj/arrowDown.png +0 -0
- pyworkflow/resources/showj/arrowUp.png +0 -0
- pyworkflow/resources/showj/background_section.png +0 -0
- pyworkflow/resources/showj/colRowModeOff.png +0 -0
- pyworkflow/resources/showj/colRowModeOn.png +0 -0
- pyworkflow/resources/showj/delete.png +0 -0
- pyworkflow/resources/showj/doc_icon.png +0 -0
- pyworkflow/resources/showj/download_icon.png +0 -0
- pyworkflow/resources/showj/enabled_gallery.png +0 -0
- pyworkflow/resources/showj/galleryViewOff.png +0 -0
- pyworkflow/resources/showj/galleryViewOn.png +0 -0
- pyworkflow/resources/showj/goto.png +0 -0
- pyworkflow/resources/showj/menu.png +0 -0
- pyworkflow/resources/showj/separator.png +0 -0
- pyworkflow/resources/showj/tableViewOff.png +0 -0
- pyworkflow/resources/showj/tableViewOn.png +0 -0
- pyworkflow/resources/showj/ui-bg_glass_75_e6e6e6_1x400.png +0 -0
- pyworkflow/resources/showj/ui-bg_glass_95_fef1ec_1x400.png +0 -0
- pyworkflow/resources/showj/ui-bg_highlight-soft_75_cccccc_1x100.png +0 -0
- pyworkflow/resources/showj/volumeOff.png +0 -0
- pyworkflow/resources/showj/volumeOn.png +0 -0
- pyworkflow/viewer.py +23 -1
- pyworkflowtests/objects.py +2 -2
- pyworkflowtests/protocols.py +1 -3
- {scipion_pyworkflow-3.10.5.dist-info → scipion_pyworkflow-3.11.0.dist-info}/METADATA +21 -25
- scipion_pyworkflow-3.11.0.dist-info/RECORD +71 -0
- {scipion_pyworkflow-3.10.5.dist-info → scipion_pyworkflow-3.11.0.dist-info}/WHEEL +1 -1
- scipion_pyworkflow-3.11.0.dist-info/entry_points.txt +2 -0
- pyworkflow/apps/__init__.py +0 -29
- pyworkflow/apps/pw_manager.py +0 -37
- pyworkflow/apps/pw_plot.py +0 -51
- pyworkflow/apps/pw_project.py +0 -113
- pyworkflow/apps/pw_protocol_list.py +0 -143
- pyworkflow/apps/pw_protocol_run.py +0 -51
- pyworkflow/apps/pw_run_tests.py +0 -267
- pyworkflow/apps/pw_schedule_run.py +0 -322
- pyworkflow/apps/pw_sleep.py +0 -37
- pyworkflow/apps/pw_sync_data.py +0 -439
- pyworkflow/apps/pw_viewer.py +0 -78
- pyworkflow/gui/__init__.py +0 -36
- pyworkflow/gui/browser.py +0 -726
- pyworkflow/gui/canvas.py +0 -1190
- pyworkflow/gui/dialog.py +0 -977
- pyworkflow/gui/form.py +0 -2637
- pyworkflow/gui/graph.py +0 -247
- pyworkflow/gui/graph_layout.py +0 -271
- pyworkflow/gui/gui.py +0 -566
- pyworkflow/gui/matplotlib_image.py +0 -233
- pyworkflow/gui/plotter.py +0 -247
- pyworkflow/gui/project/__init__.py +0 -25
- pyworkflow/gui/project/base.py +0 -192
- pyworkflow/gui/project/constants.py +0 -139
- pyworkflow/gui/project/labels.py +0 -205
- pyworkflow/gui/project/project.py +0 -492
- pyworkflow/gui/project/searchprotocol.py +0 -154
- pyworkflow/gui/project/searchrun.py +0 -181
- pyworkflow/gui/project/steps.py +0 -171
- pyworkflow/gui/project/utils.py +0 -332
- pyworkflow/gui/project/variables.py +0 -179
- pyworkflow/gui/project/viewdata.py +0 -472
- pyworkflow/gui/project/viewprojects.py +0 -510
- pyworkflow/gui/project/viewprotocols.py +0 -2093
- pyworkflow/gui/project/viewprotocols_extra.py +0 -560
- pyworkflow/gui/text.py +0 -771
- pyworkflow/gui/tooltip.py +0 -185
- pyworkflow/gui/tree.py +0 -684
- pyworkflow/gui/widgets.py +0 -307
- pyworkflow/mapper/__init__.py +0 -26
- pyworkflow/mapper/mapper.py +0 -222
- pyworkflow/mapper/sqlite.py +0 -1578
- pyworkflow/mapper/sqlite_db.py +0 -145
- pyworkflow/project/__init__.py +0 -31
- pyworkflow/project/config.py +0 -454
- pyworkflow/project/manager.py +0 -180
- pyworkflow/project/project.py +0 -2010
- pyworkflow/protocol/__init__.py +0 -38
- pyworkflow/protocol/bibtex.py +0 -48
- pyworkflow/protocol/constants.py +0 -87
- pyworkflow/protocol/executor.py +0 -455
- pyworkflow/protocol/hosts.py +0 -313
- pyworkflow/protocol/launch.py +0 -270
- pyworkflow/protocol/package.py +0 -42
- pyworkflow/protocol/params.py +0 -741
- pyworkflow/protocol/protocol.py +0 -2582
- pyworkflow/tests/__init__.py +0 -29
- pyworkflow/tests/test_utils.py +0 -25
- pyworkflow/tests/tests.py +0 -341
- pyworkflow/utils/__init__.py +0 -38
- pyworkflow/utils/dataset.py +0 -414
- pyworkflow/utils/echo.py +0 -104
- pyworkflow/utils/graph.py +0 -169
- pyworkflow/utils/log.py +0 -284
- pyworkflow/utils/path.py +0 -528
- pyworkflow/utils/process.py +0 -132
- pyworkflow/utils/profiler.py +0 -92
- pyworkflow/utils/progressbar.py +0 -154
- pyworkflow/utils/properties.py +0 -631
- pyworkflow/utils/reflection.py +0 -129
- pyworkflow/utils/utils.py +0 -879
- pyworkflow/utils/which.py +0 -229
- pyworkflow/webservices/__init__.py +0 -8
- pyworkflow/webservices/config.py +0 -11
- pyworkflow/webservices/notifier.py +0 -162
- pyworkflow/webservices/repository.py +0 -59
- pyworkflow/webservices/workflowhub.py +0 -74
- pyworkflowtests/tests/__init__.py +0 -0
- pyworkflowtests/tests/test_canvas.py +0 -72
- pyworkflowtests/tests/test_domain.py +0 -45
- pyworkflowtests/tests/test_logs.py +0 -74
- pyworkflowtests/tests/test_mappers.py +0 -392
- pyworkflowtests/tests/test_object.py +0 -507
- pyworkflowtests/tests/test_project.py +0 -42
- pyworkflowtests/tests/test_protocol_execution.py +0 -135
- pyworkflowtests/tests/test_protocol_export.py +0 -78
- pyworkflowtests/tests/test_protocol_output.py +0 -158
- pyworkflowtests/tests/test_streaming.py +0 -47
- pyworkflowtests/tests/test_utils.py +0 -210
- scipion_pyworkflow-3.10.5.dist-info/RECORD +0 -140
- scipion_pyworkflow-3.10.5.dist-info/dependency_links.txt +0 -1
- scipion_pyworkflow-3.10.5.dist-info/entry_points.txt +0 -5
- {scipion_pyworkflow-3.10.5.dist-info → scipion_pyworkflow-3.11.0.dist-info/licenses}/LICENSE.txt +0 -0
- {scipion_pyworkflow-3.10.5.dist-info → scipion_pyworkflow-3.11.0.dist-info}/top_level.txt +0 -0
@@ -1,507 +0,0 @@
|
|
1
|
-
#!/usr/bin/env python
|
2
|
-
# -*- coding: utf-8 -*-
|
3
|
-
# **************************************************************************
|
4
|
-
# *
|
5
|
-
# * Authors: J.M. De la Rosa Trevin (delarosatrevin@scilifelab.se) [1]
|
6
|
-
# *
|
7
|
-
# * [1] SciLifeLab, Stockholm University
|
8
|
-
# *
|
9
|
-
# * This program is free software: you can redistribute it and/or modify
|
10
|
-
# * it under the terms of the GNU General Public License as published by
|
11
|
-
# * the Free Software Foundation, either version 3 of the License, or
|
12
|
-
# * (at your option) any later version.
|
13
|
-
# *
|
14
|
-
# * This program is distributed in the hope that it will be useful,
|
15
|
-
# * but WITHOUT ANY WARRANTY; without even the implied warranty of
|
16
|
-
# * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
17
|
-
# * GNU General Public License for more details.
|
18
|
-
# *
|
19
|
-
# * You should have received a copy of the GNU General Public License
|
20
|
-
# * along with this program. If not, see <https://www.gnu.org/licenses/>.
|
21
|
-
# *
|
22
|
-
# * All comments concerning this program package may be sent to the
|
23
|
-
# * e-mail address 'scipion@cnb.csic.es'
|
24
|
-
# *
|
25
|
-
# **************************************************************************
|
26
|
-
import datetime as dt
|
27
|
-
try:
|
28
|
-
from datetime import UTC # python 3.11+
|
29
|
-
except ImportError:
|
30
|
-
UTC = dt.timezone.utc
|
31
|
-
import os
|
32
|
-
from logging import DEBUG, lastResort
|
33
|
-
from time import sleep
|
34
|
-
|
35
|
-
import pyworkflow.object as pwobj
|
36
|
-
import pyworkflow.tests as pwtests
|
37
|
-
from pyworkflow.mapper.sqlite import ID, CREATION
|
38
|
-
from ..objects import (Complex, MockSetOfImages, MockImage, MockObject,
|
39
|
-
MockAcquisition, MockMicrograph)
|
40
|
-
|
41
|
-
IMAGES_STK = "images.stk"
|
42
|
-
|
43
|
-
NUMERIC_ATRIBUTE_NAME = "1"
|
44
|
-
|
45
|
-
NUMERIC_ATTRIBUTE_VALUE = "numeric_attribute"
|
46
|
-
|
47
|
-
|
48
|
-
class ListContainer(pwobj.Object):
|
49
|
-
def __init__(self, **args):
|
50
|
-
pwobj.Object.__init__(self, **args)
|
51
|
-
self.csv = pwobj.CsvList()
|
52
|
-
|
53
|
-
|
54
|
-
class TestObject(pwtests.BaseTest):
|
55
|
-
@classmethod
|
56
|
-
def setUpClass(cls):
|
57
|
-
pwtests.setupTestOutput(cls)
|
58
|
-
|
59
|
-
def test_ObjectsDict(self):
|
60
|
-
# Validate that the object dict is populated correctly
|
61
|
-
basicObjNames = [
|
62
|
-
'Scalar', 'Integer', 'Float', 'String', 'Pointer', 'Boolean',
|
63
|
-
'OrderedObject', 'List', 'CsvList', 'PointerList', 'Set'
|
64
|
-
]
|
65
|
-
self.assertTrue(all(name in pwobj.OBJECTS_DICT
|
66
|
-
for name in basicObjNames))
|
67
|
-
|
68
|
-
def test_Object(self):
|
69
|
-
value = 2
|
70
|
-
i = pwobj.Integer(value)
|
71
|
-
self.assertEqual(value, i.get())
|
72
|
-
# compare objects
|
73
|
-
i2 = pwobj.Integer(value)
|
74
|
-
self.assertEqual(i, i2)
|
75
|
-
|
76
|
-
value = 2.
|
77
|
-
f = pwobj.Float(value)
|
78
|
-
self.assertAlmostEqual(value, f.get())
|
79
|
-
|
80
|
-
f.multiply(5)
|
81
|
-
self.assertAlmostEqual(value*5, f.get())
|
82
|
-
|
83
|
-
a = pwobj.Integer()
|
84
|
-
self.assertEqual(a.hasValue(), False)
|
85
|
-
c = Complex.createComplex()
|
86
|
-
# Check values are correct
|
87
|
-
self.assertEqual(c.imag.get(), Complex.cGold.imag)
|
88
|
-
self.assertEqual(c.real.get(), Complex.cGold.real)
|
89
|
-
|
90
|
-
# Test Boolean logic
|
91
|
-
b = pwobj.Boolean(False)
|
92
|
-
self.assertTrue(not b.get())
|
93
|
-
|
94
|
-
b.set('True')
|
95
|
-
self.assertTrue(b.get())
|
96
|
-
|
97
|
-
b = pwobj.Boolean()
|
98
|
-
b.set(False)
|
99
|
-
self.assertTrue(not b.get())
|
100
|
-
|
101
|
-
# CsvList should be empty if set to ''
|
102
|
-
l = pwobj.CsvList()
|
103
|
-
l.set('')
|
104
|
-
self.assertEqual(len(l), 0)
|
105
|
-
|
106
|
-
# Test emptiness
|
107
|
-
self.assertIsNotEmpty(b)
|
108
|
-
|
109
|
-
def testWithPointer(self):
|
110
|
-
|
111
|
-
obj = pwobj.Integer(10)
|
112
|
-
|
113
|
-
self.assertFalse(obj.hasPointer(),
|
114
|
-
"Default instantiation off Integer has a pointer.")
|
115
|
-
|
116
|
-
self.assertEqual(obj.get(), 10, "Integer.get(), without pointer fails.")
|
117
|
-
|
118
|
-
pointee = pwobj.Object()
|
119
|
-
setattr(pointee, "value", pwobj.Integer(20))
|
120
|
-
|
121
|
-
# Set a pointer (not a real case though, but enough here)
|
122
|
-
obj.setPointer(pwobj.Pointer(pointee, extended='value'))
|
123
|
-
|
124
|
-
self.assertEqual(obj.get(), 20, "Integer.get() fails with a pointer.")
|
125
|
-
|
126
|
-
def test_String(self):
|
127
|
-
value = 'thisisastring'
|
128
|
-
s = pwobj.String(value)
|
129
|
-
self.assertEqual(value, s.get())
|
130
|
-
self.assertEqual(s.hasValue(), True)
|
131
|
-
|
132
|
-
s2 = pwobj.String()
|
133
|
-
# None value is considered empty
|
134
|
-
self.assertTrue(s2.empty(), "s2 string should be empty if None")
|
135
|
-
s2.set(' ')
|
136
|
-
# Only spaces is also empty
|
137
|
-
self.assertTrue(s2.empty(), "s2 string should be empty if only spaces")
|
138
|
-
s2.set('something')
|
139
|
-
# No empty after some value
|
140
|
-
self.assertFalse(s2.empty(),
|
141
|
-
"s2 string should not be empty after value")
|
142
|
-
|
143
|
-
now = dt.datetime.now()
|
144
|
-
s.set(now)
|
145
|
-
self.assertEqual(now, s.datetime())
|
146
|
-
|
147
|
-
# Ranges and values
|
148
|
-
s2.set("1 2 3 4")
|
149
|
-
self.assertEqual(s2.getListFromValues(caster=float), [1.,2.,3.,4.])
|
150
|
-
self.assertEqual(s2.getListFromRange(), [1, 2, 3, 4])
|
151
|
-
|
152
|
-
# Values ...
|
153
|
-
s2.set("2x4, 4, 7")
|
154
|
-
self.assertEqual(s2.getListFromValues(), [4, 4, 4, 7])
|
155
|
-
|
156
|
-
# Values ...
|
157
|
-
self.assertEqual(s2.getListFromValues(caster=str), ["2x4", "4", "7"])
|
158
|
-
|
159
|
-
# Ranges
|
160
|
-
s2.set("2-8, 1-2, 7")
|
161
|
-
self.assertEqual(s2.getListFromRange(), [2, 3, 4, 5, 6, 7, 8, 1, 2, 7])
|
162
|
-
|
163
|
-
def test_Pointer(self):
|
164
|
-
c = Complex.createComplex()
|
165
|
-
p = pwobj.Pointer()
|
166
|
-
p.set(c)
|
167
|
-
p.setExtended('Name')
|
168
|
-
c.Name = pwobj.String('Paquito')
|
169
|
-
|
170
|
-
self.assertEqual(p.get(), 'Paquito')
|
171
|
-
stackFn = IMAGES_STK
|
172
|
-
mrcsFn = "images.mrcs"
|
173
|
-
fn = self.getOutputPath('test_images.sqlite')
|
174
|
-
imgSet = MockSetOfImages(filename=fn)
|
175
|
-
imgSet.setSamplingRate(1.0)
|
176
|
-
for i in range(10):
|
177
|
-
img = MockImage()
|
178
|
-
img.setLocation(i+1, stackFn)
|
179
|
-
imgSet.append(img)
|
180
|
-
|
181
|
-
imgSet.write()
|
182
|
-
|
183
|
-
# Test that image number 7 is correctly retrieved
|
184
|
-
# from the set
|
185
|
-
img7 = imgSet[7]
|
186
|
-
self.assertEqual(img7.getFileName(), stackFn)
|
187
|
-
|
188
|
-
# Modify some properties of image 7 to test update
|
189
|
-
img7.setFileName(mrcsFn)
|
190
|
-
img7.setSamplingRate(2.0)
|
191
|
-
imgSet.update(img7)
|
192
|
-
# Write changes after the image 7 update
|
193
|
-
imgSet.write()
|
194
|
-
|
195
|
-
# Read again the set to be able to retrieve elements
|
196
|
-
imgSet = MockSetOfImages(filename=fn)
|
197
|
-
|
198
|
-
# Validate that image7 was properly updated
|
199
|
-
img7 = imgSet[7]
|
200
|
-
self.assertEqual(img7.getFileName(), mrcsFn)
|
201
|
-
|
202
|
-
o = MockObject()
|
203
|
-
|
204
|
-
o.pointer = pwobj.Pointer()
|
205
|
-
o.pointer.set(imgSet)
|
206
|
-
|
207
|
-
# This is not true anymore and is allowed unless we see is needed
|
208
|
-
# The main reason is a boost in performance.
|
209
|
-
# o.refC = o.pointer.get()
|
210
|
-
# attrNames = [k for k, a in o.getAttributes()]
|
211
|
-
# # Check that 'refC' should not appear in attributes
|
212
|
-
# # since it is only an "alias" to an existing pointed value
|
213
|
-
# self.assertNotIn('refC', attrNames)
|
214
|
-
|
215
|
-
self.assertFalse(o.pointer.hasExtended(),
|
216
|
-
'o.pointer should not have extended at this point')
|
217
|
-
|
218
|
-
o.pointer.setExtended(7)
|
219
|
-
|
220
|
-
self.assertTrue(o.pointer.hasExtended())
|
221
|
-
self.assertTrue(o.pointer.hasExtended())
|
222
|
-
self.assertEqual(o.pointer.getExtended(), "7")
|
223
|
-
|
224
|
-
# Check that the Item 7 of the set is properly
|
225
|
-
# retrieved by the pointer after setting the extended to 7
|
226
|
-
self.assertEqual(imgSet[7].getObjId(), o.pointer.get().getObjId())
|
227
|
-
|
228
|
-
# Test the keyword arguments of Pointer constructor
|
229
|
-
# repeat above tests with new pointer
|
230
|
-
ptr = pwobj.Pointer(value=imgSet, extended=7)
|
231
|
-
self.assertTrue(ptr.hasExtended())
|
232
|
-
self.assertTrue(ptr.hasExtended())
|
233
|
-
self.assertEqual(ptr.getExtended(), "7")
|
234
|
-
|
235
|
-
# Check that the Item 7 of the set is properly
|
236
|
-
# retrieved by the pointer after setting the extended to 7
|
237
|
-
self.assertEqual(imgSet[7], ptr.get())
|
238
|
-
|
239
|
-
o2 = pwobj.Object()
|
240
|
-
o2.outputImages = imgSet
|
241
|
-
ptr2 = pwobj.Pointer()
|
242
|
-
ptr2.set(o2)
|
243
|
-
# Test nested extended attributes
|
244
|
-
ptr2.setExtended('outputImages.7')
|
245
|
-
self.assertEqual(imgSet[7], ptr2.get())
|
246
|
-
|
247
|
-
# Same as ptr2, but setting extended in constructor
|
248
|
-
ptr3 = pwobj.Pointer(value=o2, extended='outputImages.7')
|
249
|
-
self.assertEqual(imgSet[7], ptr3.get())
|
250
|
-
|
251
|
-
# Test copy between pointer objects
|
252
|
-
ptr4 = pwobj.Pointer()
|
253
|
-
ptr4.copy(ptr3)
|
254
|
-
self.assertEqual(imgSet[7], ptr4.get())
|
255
|
-
self.assertEqual(ptr4.getExtended(), 'outputImages.7')
|
256
|
-
|
257
|
-
# Test numeric attributes.
|
258
|
-
setattr(o2, NUMERIC_ATRIBUTE_NAME, NUMERIC_ATTRIBUTE_VALUE)
|
259
|
-
ptr5 = pwobj.Pointer(value=o2, extended=NUMERIC_ATRIBUTE_NAME)
|
260
|
-
self.assertEqual(NUMERIC_ATTRIBUTE_VALUE, ptr5.get())
|
261
|
-
|
262
|
-
def test_Sets(self):
|
263
|
-
stackFn = IMAGES_STK
|
264
|
-
fn = self.getOutputPath('test_images2.sqlite')
|
265
|
-
|
266
|
-
imgSet = MockSetOfImages(filename=fn)
|
267
|
-
|
268
|
-
halfTimeStamp = None
|
269
|
-
|
270
|
-
for i in range(10):
|
271
|
-
img = MockImage()
|
272
|
-
img.setLocation(i + 1, stackFn)
|
273
|
-
img.setSamplingRate(i % 3)
|
274
|
-
imgSet.append(img)
|
275
|
-
if i == 4:
|
276
|
-
sleep(1)
|
277
|
-
halfTimeStamp = dt.datetime.now(UTC).replace(microsecond=0)
|
278
|
-
imgSet.write()
|
279
|
-
|
280
|
-
# Test size is 10
|
281
|
-
self.assertSetSize(imgSet, 10)
|
282
|
-
|
283
|
-
# Test hasChangedSince
|
284
|
-
timeStamp = dt.datetime.now()
|
285
|
-
self.assertFalse(imgSet.hasChangedSince(timeStamp), "Set.hasChangedSince returns true when it hasn't changed.")
|
286
|
-
# Remove 10 seconds
|
287
|
-
self.assertTrue(imgSet.hasChangedSince(timeStamp-dt.timedelta(0,10)), "Set.hasChangedSince returns false when it has changed.")
|
288
|
-
|
289
|
-
# PERFORMANCE functionality
|
290
|
-
def checkSetIteration(limit, skipRows=None):
|
291
|
-
|
292
|
-
expectedId = 1 if skipRows is None else skipRows+1
|
293
|
-
index = 0
|
294
|
-
for item in imgSet.iterItems(limit=(limit, skipRows)):
|
295
|
-
self.assertEqual(item.getIndex(), expectedId+index,
|
296
|
-
"Wrong item in set when using limits.")
|
297
|
-
index += 1
|
298
|
-
|
299
|
-
self.assertEqual(index, limit,
|
300
|
-
"Number of iterations wrong with limits")
|
301
|
-
|
302
|
-
# Check iteration with limit
|
303
|
-
checkSetIteration(2)
|
304
|
-
|
305
|
-
# Check iteration with limit and skip rows
|
306
|
-
checkSetIteration(3, 2)
|
307
|
-
|
308
|
-
# Tests unique method
|
309
|
-
# Requesting 1 unique value as string
|
310
|
-
result = imgSet.getUniqueValues("_samplingRate")
|
311
|
-
self.assertEqual(len(result), 3, "Unique values wrong for 1 attribute and 3 value")
|
312
|
-
|
313
|
-
# Requesting 1 unique value as list
|
314
|
-
result = imgSet.getUniqueValues(["_samplingRate"])
|
315
|
-
self.assertEqual(len(result), 3, "Unique values wrong for 1 attribute and one value as list")
|
316
|
-
|
317
|
-
# Requesting several unique values as string
|
318
|
-
result = imgSet.getUniqueValues("_index")
|
319
|
-
self.assertEqual(len(result), 10, "Unique values wrong for id attribute")
|
320
|
-
|
321
|
-
# Requesting several unique values with several columns
|
322
|
-
result = imgSet.getUniqueValues(["_filename", "_samplingRate"])
|
323
|
-
# Here we should have 2 keys containing 2 list
|
324
|
-
self.assertEqual(len(result), 2, "Unique values dictionary length wrong")
|
325
|
-
self.assertEqual(len(result["_filename"]), 3, "Unique values dict item size wrong")
|
326
|
-
|
327
|
-
# Requesting unique values with where
|
328
|
-
result = imgSet.getUniqueValues("_index",where="_samplingRate = 2")
|
329
|
-
# Here we should have 2 values
|
330
|
-
self.assertEqual(len(result), 3, "Unique values with filter not working")
|
331
|
-
|
332
|
-
# Request id list
|
333
|
-
result = imgSet.getUniqueValues(ID)
|
334
|
-
# Here we should have 10 values
|
335
|
-
self.assertEqual(len(result), 10, "Unique values with ID")
|
336
|
-
|
337
|
-
# Use creation timestamp
|
338
|
-
# Request id list
|
339
|
-
result = imgSet.getUniqueValues(ID, where="%s>=%s" % (CREATION, imgSet.fmtDate(halfTimeStamp)))
|
340
|
-
self.assertEqual(len(result), 5, "Unique values after a time stamp does not work")
|
341
|
-
|
342
|
-
# Test getIdSet
|
343
|
-
ids = imgSet.getIdSet()
|
344
|
-
self.assertIsInstance(ids, set, "getIdSet does not return a set")
|
345
|
-
self.assertIsInstance(next(iter(ids)), int, "getIdSet items are not integer")
|
346
|
-
self.assertEqual(len(ids), 10, "getIdSet does not return 10 items")
|
347
|
-
|
348
|
-
# Request item by id
|
349
|
-
item = imgSet[1]
|
350
|
-
self.assertEqual(item.getObjId(), 1, "Item accessed by [] and id does not work")
|
351
|
-
|
352
|
-
# Request item by field
|
353
|
-
item = imgSet.getItem("id", 2)
|
354
|
-
self.assertEqual(item.getObjId(), 2, "Item accessed field id does not work")
|
355
|
-
|
356
|
-
# Test load properties queries
|
357
|
-
from pyworkflow.mapper.sqlite_db import logger
|
358
|
-
logger.setLevel(DEBUG)
|
359
|
-
lastResort.setLevel(DEBUG)
|
360
|
-
imgSetVerbose = MockSetOfImages(filename=fn)
|
361
|
-
imgSetVerbose.loadAllProperties()
|
362
|
-
|
363
|
-
# Compare sets are "equal"
|
364
|
-
self.compareSetProperties(imgSet, imgSetVerbose, ignore=[])
|
365
|
-
|
366
|
-
|
367
|
-
def test_copyAttributes(self):
|
368
|
-
""" Check that after copyAttributes, the values
|
369
|
-
were properly copied.
|
370
|
-
"""
|
371
|
-
c1 = Complex(imag=10., real=11.)
|
372
|
-
c2 = Complex(imag=0., real=1.0001)
|
373
|
-
|
374
|
-
# Float values are different, should not be equal
|
375
|
-
self.assertFalse(c1.equalAttributes(c2))
|
376
|
-
c2.copyAttributes(c1, 'imag', 'real')
|
377
|
-
|
378
|
-
self.assertTrue(c1.equalAttributes(c2),
|
379
|
-
'Complex c1 and c2 have not equal attributes'
|
380
|
-
'\nc1: %s\nc2: %s\n' % (c1, c2))
|
381
|
-
|
382
|
-
c1.score = pwobj.Float(1.)
|
383
|
-
|
384
|
-
# If we copyAttributes again, score dynamic attribute should
|
385
|
-
# be set in c2
|
386
|
-
c2.copyAttributes(c1, 'score')
|
387
|
-
self.assertTrue(hasattr(c2, 'score'))
|
388
|
-
|
389
|
-
def test_equalAttributes(self):
|
390
|
-
""" Check that equal attributes function behaves well
|
391
|
-
to compare floats with a given precision.
|
392
|
-
"""
|
393
|
-
c1 = Complex(imag=0., real=1.)
|
394
|
-
c2 = Complex(imag=0., real=1.0001)
|
395
|
-
|
396
|
-
# Since Float precision is 0.001, now c1 and c2
|
397
|
-
# should have equal attributes
|
398
|
-
self.assertTrue(c1.equalAttributes(c2))
|
399
|
-
# Now if we set a more restrictive precision
|
400
|
-
# c1 and c2 are not longer equals
|
401
|
-
pwobj.Float.setPrecision(0.0000001)
|
402
|
-
self.assertFalse(c1.equalAttributes(c2))
|
403
|
-
|
404
|
-
def test_formatString(self):
|
405
|
-
""" Test that Scalar objects behave well
|
406
|
-
when using string formatting such as: %f or %d
|
407
|
-
"""
|
408
|
-
i = pwobj.Integer(10)
|
409
|
-
f = pwobj.Float(3.345)
|
410
|
-
|
411
|
-
s1 = "i = %d, f = %0.3f" % (i, f)
|
412
|
-
|
413
|
-
self.assertEqual(s1, "i = 10, f = 3.345")
|
414
|
-
|
415
|
-
def test_getObjDict(self):
|
416
|
-
""" Test retrieving an object dictionary with its attribute values."""
|
417
|
-
acq1 = MockAcquisition(magnification=50000,
|
418
|
-
voltage=200,
|
419
|
-
sphericalAberration=2.7,
|
420
|
-
dosePerFrame=1)
|
421
|
-
m1 = MockMicrograph(
|
422
|
-
'my_movie.mrc', objId=1, objLabel='test micrograph',
|
423
|
-
objComment='Testing store and retrieve from dict.')
|
424
|
-
m1.setSamplingRate(1.6)
|
425
|
-
m1.setAcquisition(acq1)
|
426
|
-
m1Dict = m1.getObjDict(includeBasic=True)
|
427
|
-
|
428
|
-
goldDict1 = dict([
|
429
|
-
('object.id', 1),
|
430
|
-
('object.label', 'test micrograph'),
|
431
|
-
('object.comment', 'Testing store and retrieve from dict.'),
|
432
|
-
('_index', 0),
|
433
|
-
('_filename', 'my_movie.mrc'),
|
434
|
-
('_samplingRate', 1.6),
|
435
|
-
('_micName', None),
|
436
|
-
('_acquisition', None),
|
437
|
-
('_acquisition._magnification', 50000.0),
|
438
|
-
('_acquisition._voltage', 200.0),
|
439
|
-
('_acquisition._sphericalAberration', 2.7),
|
440
|
-
('_acquisition._amplitudeContrast', None),
|
441
|
-
('_acquisition._doseInitial', 0.0),
|
442
|
-
('_acquisition._dosePerFrame', 1.0),
|
443
|
-
])
|
444
|
-
|
445
|
-
self.assertEqual(goldDict1, m1Dict)
|
446
|
-
|
447
|
-
def test_Dict(self):
|
448
|
-
d = pwobj.Dict(default='missing')
|
449
|
-
d.update({1: 'one', 2: 'two'})
|
450
|
-
|
451
|
-
# Return default value for any non-present key
|
452
|
-
self.assertEqual('missing', d[10])
|
453
|
-
|
454
|
-
# Return true for any 'contains' query
|
455
|
-
self.assertTrue(100 in d)
|
456
|
-
|
457
|
-
|
458
|
-
class TestUtils(pwtests.BaseTest):
|
459
|
-
|
460
|
-
@classmethod
|
461
|
-
def setUpClass(cls):
|
462
|
-
pwtests.setupTestOutput(cls)
|
463
|
-
|
464
|
-
def test_ListsFunctions(self):
|
465
|
-
""" Test of some methods that retrieve lists from string. """
|
466
|
-
from pyworkflow.utils import getListFromValues, getFloatListFromValues,\
|
467
|
-
getBoolListFromValues
|
468
|
-
|
469
|
-
results = [('2x1 2x2 4 5', getListFromValues, ['2x1', '2x2', '4', '5'], None),
|
470
|
-
('2x1 2x2 4 5', getFloatListFromValues, [1., 1., 2., 2., 4., 5.],None),
|
471
|
-
('1 2 3x3 0.5', getFloatListFromValues, [1., 2., 3., 3., 3., 0.5, 0.5, 0.5], 8),
|
472
|
-
('3x1 3x0 1', getBoolListFromValues, [True, True, True, False, False, False, True, True],8),
|
473
|
-
]
|
474
|
-
|
475
|
-
for s, func, goldList, length in results:
|
476
|
-
|
477
|
-
l = func(s, length=length)
|
478
|
-
for i in range(0,len(goldList)):
|
479
|
-
self.assertEqual(l[i], goldList[i])
|
480
|
-
|
481
|
-
self.assertEqual(len(goldList), len(l), )
|
482
|
-
|
483
|
-
def test_Environ(self):
|
484
|
-
""" Test the Environ class with its utilities. """
|
485
|
-
from pyworkflow.utils import Environ
|
486
|
-
env = Environ({'PATH': '/usr/bin:/usr/local/bin',
|
487
|
-
'LD_LIBRARY_PATH': '/usr/lib:/usr/lib64'
|
488
|
-
})
|
489
|
-
env1 = Environ(env)
|
490
|
-
env1.set('PATH', '/usr/local/xmipp')
|
491
|
-
self.assertEqual(env1['PATH'], '/usr/local/xmipp')
|
492
|
-
self.assertEqual(env1['LD_LIBRARY_PATH'], env['LD_LIBRARY_PATH'])
|
493
|
-
|
494
|
-
env2 = Environ(env)
|
495
|
-
env2.set('PATH', '/usr/local/xmipp', position=Environ.BEGIN)
|
496
|
-
self.assertEqual(env2['PATH'], '/usr/local/xmipp' + os.pathsep +
|
497
|
-
env['PATH'])
|
498
|
-
self.assertEqual(env2['LD_LIBRARY_PATH'], env['LD_LIBRARY_PATH'])
|
499
|
-
|
500
|
-
env3 = Environ(env)
|
501
|
-
env3.update({'PATH': '/usr/local/xmipp',
|
502
|
-
'LD_LIBRARY_PATH': '/usr/local/xmipp/lib'},
|
503
|
-
position=Environ.END)
|
504
|
-
self.assertEqual(env3['PATH'], env['PATH'] + os.pathsep +
|
505
|
-
'/usr/local/xmipp')
|
506
|
-
self.assertEqual(env3['LD_LIBRARY_PATH'], env['LD_LIBRARY_PATH'] +
|
507
|
-
os.pathsep + '/usr/local/xmipp/lib')
|
@@ -1,42 +0,0 @@
|
|
1
|
-
#!/usr/bin/env python
|
2
|
-
# **************************************************************************
|
3
|
-
# *
|
4
|
-
# * Authors: Pablo Conesa (pconesa@cnb.csic.es) [1]
|
5
|
-
# *
|
6
|
-
# * [1] Unidad de Bioinformatica of Centro Nacional de Biotecnologia , CSIC
|
7
|
-
# *
|
8
|
-
# * This program is free software: you can redistribute it and/or modify
|
9
|
-
# * it under the terms of the GNU General Public License as published by
|
10
|
-
# * the Free Software Foundation, either version 3 of the License, or
|
11
|
-
# * (at your option) any later version.
|
12
|
-
# *
|
13
|
-
# * This program is distributed in the hope that it will be useful,
|
14
|
-
# * but WITHOUT ANY WARRANTY; without even the implied warranty of
|
15
|
-
# * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
16
|
-
# * GNU General Public License for more details.
|
17
|
-
# *
|
18
|
-
# * You should have received a copy of the GNU General Public License
|
19
|
-
# * along with this program. If not, see <https://www.gnu.org/licenses/>.
|
20
|
-
# *
|
21
|
-
# * All comments concerning this program package may be sent to the
|
22
|
-
# * e-mail address 'scipion@cnb.csic.es'
|
23
|
-
# *
|
24
|
-
# **************************************************************************
|
25
|
-
|
26
|
-
from pyworkflow.project.project import Project
|
27
|
-
from unittest import TestCase
|
28
|
-
from unittest.mock import patch
|
29
|
-
|
30
|
-
|
31
|
-
# NOTE: This test as it is might serve as a skeleton for future testing the fixLinks
|
32
|
-
# but as it is now it does not test anything at all. I leave it as an example of using patch (mock testing)
|
33
|
-
class TestProject(TestCase):
|
34
|
-
|
35
|
-
def test_fixlinks(self):
|
36
|
-
""" Test fixlinks call."""
|
37
|
-
|
38
|
-
with patch("pyworkflow.project.Project.getRuns") as getruns:
|
39
|
-
|
40
|
-
getruns.return_value = []
|
41
|
-
proj = Project("domain", "path")
|
42
|
-
proj.fixLinks("foo")
|
@@ -1,135 +0,0 @@
|
|
1
|
-
# -*- coding: utf-8 -*-
|
2
|
-
# **************************************************************************
|
3
|
-
# *
|
4
|
-
# * Authors: J.M. De la Rosa Trevin (delarosatrevin@scilifelab.se) [1]
|
5
|
-
# *
|
6
|
-
# * [1] SciLifeLab, Stockholm University
|
7
|
-
# *
|
8
|
-
# * This program is free software: you can redistribute it and/or modify
|
9
|
-
# * it under the terms of the GNU General Public License as published by
|
10
|
-
# * the Free Software Foundation, either version 3 of the License, or
|
11
|
-
# * (at your option) any later version.
|
12
|
-
# *
|
13
|
-
# * This program is distributed in the hope that it will be useful,
|
14
|
-
# * but WITHOUT ANY WARRANTY; without even the implied warranty of
|
15
|
-
# * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
16
|
-
# * GNU General Public License for more details.
|
17
|
-
# *
|
18
|
-
# * You should have received a copy of the GNU General Public License
|
19
|
-
# * along with this program. If not, see <https://www.gnu.org/licenses/>.
|
20
|
-
# *
|
21
|
-
# * All comments concerning this program package may be sent to the
|
22
|
-
# * e-mail address 'scipion@cnb.csic.es'
|
23
|
-
# *
|
24
|
-
# **************************************************************************
|
25
|
-
import threading
|
26
|
-
|
27
|
-
import pyworkflow.tests as pwtests
|
28
|
-
import pyworkflow.mapper as pwmapper
|
29
|
-
import pyworkflow.protocol as pwprot
|
30
|
-
from pyworkflow.project import Project
|
31
|
-
from pyworkflow.protocol.constants import VOID_GPU
|
32
|
-
|
33
|
-
|
34
|
-
# TODO: this test seems not to be finished.
|
35
|
-
from pyworkflowtests.protocols import SleepingProtocol
|
36
|
-
from pyworkflowtests import Domain
|
37
|
-
|
38
|
-
|
39
|
-
class TestProtocolExecution(pwtests.BaseTest):
|
40
|
-
|
41
|
-
@classmethod
|
42
|
-
def setUpClass(cls):
|
43
|
-
pwtests.setupTestOutput(cls)
|
44
|
-
|
45
|
-
def test_StepExecutor(self):
|
46
|
-
"""Test the list with several Complex"""
|
47
|
-
fn = self.getOutputPath("protocol.sqlite")
|
48
|
-
print("Writing to db: %s" % fn)
|
49
|
-
|
50
|
-
# Discover objects and protocols
|
51
|
-
mapperDict = Domain.getMapperDict()
|
52
|
-
|
53
|
-
# Associate the project
|
54
|
-
proj = Project(Domain, path=self.getOutputPath(''))
|
55
|
-
|
56
|
-
# Check that the protocol has associated package
|
57
|
-
mapper = pwmapper.SqliteMapper(fn, mapperDict)
|
58
|
-
prot = SleepingProtocol(mapper=mapper, n=2, project= proj,
|
59
|
-
workingDir=self.getOutputPath(''))
|
60
|
-
domain = prot.getClassDomain()
|
61
|
-
domain.printInfo()
|
62
|
-
|
63
|
-
prot.setStepsExecutor(pwprot.StepExecutor(hostConfig=None))
|
64
|
-
prot.makeWorkingDir()
|
65
|
-
prot.run()
|
66
|
-
mapper.commit()
|
67
|
-
mapper.close()
|
68
|
-
|
69
|
-
self.assertEqual(prot._steps[0].getStatus(), pwprot.STATUS_FINISHED)
|
70
|
-
|
71
|
-
mapper2 = pwmapper.SqliteMapper(fn, mapperDict)
|
72
|
-
prot2 = mapper2.selectById(prot.getObjId())
|
73
|
-
|
74
|
-
self.assertEqual(prot.endTime.get(), prot2.endTime.get())
|
75
|
-
|
76
|
-
def test_gpuSlots(self):
|
77
|
-
""" Test gpu slots are properly composed in combination of threads"""
|
78
|
-
|
79
|
-
# Test basic GPU setu methods
|
80
|
-
stepExecutor = pwprot.ThreadStepExecutor(None, 1, gpuList=None)
|
81
|
-
|
82
|
-
|
83
|
-
self.assertEqual(stepExecutor.cleanVoidGPUs([0,1]), [0,1],
|
84
|
-
"CleanVoidGpus does not work in absence of void GPUS")
|
85
|
-
|
86
|
-
self.assertEqual(stepExecutor.cleanVoidGPUs([0, VOID_GPU]), [0],
|
87
|
-
"CleanVoidGpus does not work with a void GPU")
|
88
|
-
|
89
|
-
self.assertEqual(stepExecutor.cleanVoidGPUs([VOID_GPU, VOID_GPU]), [],
|
90
|
-
"CleanVoidGpus does not work with all void GPU")
|
91
|
-
|
92
|
-
|
93
|
-
currThread = threading.currentThread()
|
94
|
-
currThread.thId = 1
|
95
|
-
self.assertEqual(stepExecutor.getGpuList(),[], "Gpu list should be empty")
|
96
|
-
|
97
|
-
# 2 threads 1 GPU
|
98
|
-
stepExecutor = pwprot.ThreadStepExecutor(None, 2, gpuList=[1])
|
99
|
-
self.assertEqual(stepExecutor.getGpuList(),[1], "Gpu list should be [1]")
|
100
|
-
|
101
|
-
currThread.thId = 2
|
102
|
-
self.assertEqual(stepExecutor.getGpuList(),[], "Gpu list should be empty after a second request")
|
103
|
-
|
104
|
-
|
105
|
-
# 2 threads 3 GPUs
|
106
|
-
stepExecutor = pwprot.ThreadStepExecutor(None, 2, gpuList=[0,1,2])
|
107
|
-
self.assertEqual(stepExecutor.getGpuList(),[0,1], "Gpu list should be [0,1]")
|
108
|
-
|
109
|
-
currThread.thId = 1
|
110
|
-
self.assertEqual(stepExecutor.getGpuList(),[2], "Gpu list should be [2] after a second request")
|
111
|
-
|
112
|
-
|
113
|
-
# 2 threads 4 GPUs with void gpus
|
114
|
-
stepExecutor = pwprot.ThreadStepExecutor(None, 2, gpuList=[0,1,2, VOID_GPU])
|
115
|
-
self.assertEqual(stepExecutor.getGpuList(),[0,1], "Gpu list should be [0,1]")
|
116
|
-
|
117
|
-
currThread.thId = 2
|
118
|
-
self.assertEqual(stepExecutor.getGpuList(),[2], "Gpu list should be [2] after a second request without the void gpu")
|
119
|
-
|
120
|
-
# less GPUs than threads. No extension should happen
|
121
|
-
stepExecutor = pwprot.ThreadStepExecutor(None, 4, gpuList=[0, VOID_GPU, 2])
|
122
|
-
self.assertEqual(stepExecutor.getGpuList(), [0], "Gpu list should not be extended")
|
123
|
-
|
124
|
-
currThread.thId = 1
|
125
|
-
self.assertEqual(stepExecutor.getGpuList(), [2],
|
126
|
-
"Gpu list should be [2] after a second request, skipping the VOID gpu")
|
127
|
-
|
128
|
-
currThread.thId = 3
|
129
|
-
self.assertEqual(stepExecutor.getGpuList(), [], "Gpu list should be empty ather all GPU slots are busy")
|
130
|
-
|
131
|
-
|
132
|
-
|
133
|
-
|
134
|
-
|
135
|
-
|