immlib 1.0.0.dev2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- immlib/__init__.py +131 -0
- immlib/_init.py +108 -0
- immlib/_version.py +235 -0
- immlib/doc/__init__.py +38 -0
- immlib/doc/_core.py +311 -0
- immlib/iolib/__init__.py +29 -0
- immlib/iolib/_core.py +720 -0
- immlib/pathlib/__init__.py +69 -0
- immlib/pathlib/_cache.py +152 -0
- immlib/pathlib/_core.py +869 -0
- immlib/pathlib/_osf.py +538 -0
- immlib/test/__init__.py +16 -0
- immlib/test/__main__.py +10 -0
- immlib/test/doc/__init__.py +6 -0
- immlib/test/doc/test_core.py +91 -0
- immlib/test/iolib/__init__.py +7 -0
- immlib/test/iolib/test_core.py +81 -0
- immlib/test/pathlib/__init__.py +11 -0
- immlib/test/pathlib/test_core.py +146 -0
- immlib/test/pathlib/test_osf.py +54 -0
- immlib/test/types/__init__.py +5 -0
- immlib/test/types/test_core.py +110 -0
- immlib/test/util/__init__.py +11 -0
- immlib/test/util/test_core.py +681 -0
- immlib/test/util/test_numeric.py +1374 -0
- immlib/test/util/test_quantity.py +218 -0
- immlib/test/util/test_url.py +51 -0
- immlib/test/workflow/__init__.py +9 -0
- immlib/test/workflow/test_core.py +418 -0
- immlib/test/workflow/test_plantype.py +248 -0
- immlib/types/__init__.py +29 -0
- immlib/types/_core.py +333 -0
- immlib/util/__init__.py +283 -0
- immlib/util/_core.py +2524 -0
- immlib/util/_numeric.py +2651 -0
- immlib/util/_quantity.py +523 -0
- immlib/util/_url.py +114 -0
- immlib/workflow/__init__.py +48 -0
- immlib/workflow/_core.py +1635 -0
- immlib/workflow/_plantype.py +334 -0
- immlib-1.0.0.dev2.dist-info/METADATA +76 -0
- immlib-1.0.0.dev2.dist-info/RECORD +45 -0
- immlib-1.0.0.dev2.dist-info/WHEEL +5 -0
- immlib-1.0.0.dev2.dist-info/licenses/LICENSE +21 -0
- immlib-1.0.0.dev2.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,418 @@
|
|
|
1
|
+
# -*- coding: utf-8 -*-
|
|
2
|
+
################################################################################
|
|
3
|
+
# immlib/test/workflow/test_core.py
|
|
4
|
+
#
|
|
5
|
+
# Tests of the core workflow module in immlib: i.e., tests for the code in the
|
|
6
|
+
# immlib.workflow._core module.
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
# Dependencies #################################################################
|
|
10
|
+
|
|
11
|
+
from unittest import TestCase
|
|
12
|
+
|
|
13
|
+
class TestWorkflowCore(TestCase):
|
|
14
|
+
"""Tests the immlib.workflow._core module."""
|
|
15
|
+
def test_calc(self):
|
|
16
|
+
from immlib.workflow import (calc, is_calc, is_calcfn)
|
|
17
|
+
from pcollections import (ldict, pdict)
|
|
18
|
+
# The calc decorator creates calculation objects.
|
|
19
|
+
@calc
|
|
20
|
+
def result(input_1, input_2=None):
|
|
21
|
+
"""Calculation for a result from input_1 and input_2.
|
|
22
|
+
|
|
23
|
+
Returns a single value, `'result'`, which is a list whose length is
|
|
24
|
+
`input_1` and whose elements are all `input_2`.
|
|
25
|
+
|
|
26
|
+
Inputs
|
|
27
|
+
------
|
|
28
|
+
input_1 : int
|
|
29
|
+
The number of elements to include in the result.
|
|
30
|
+
input_2 : object
|
|
31
|
+
The object to put in the list.
|
|
32
|
+
|
|
33
|
+
Outputs
|
|
34
|
+
-------
|
|
35
|
+
result : list
|
|
36
|
+
A list of `input_1` occurrences of `input_2`.
|
|
37
|
+
"""
|
|
38
|
+
return ([input_2] * input_1,)
|
|
39
|
+
self.assertTrue(is_calcfn(result))
|
|
40
|
+
self.assertTrue(is_calc(result.calc))
|
|
41
|
+
# Calculation objects have a number of members that keep track of the
|
|
42
|
+
# meta-data of the calculation.
|
|
43
|
+
# First is the name of the calculation--this is the name of the
|
|
44
|
+
# function.
|
|
45
|
+
c = result.calc
|
|
46
|
+
self.assertEqual(c.name, 'immlib.test.workflow.test_core.result')
|
|
47
|
+
# The inputs of the calculation are a set of the inputs of the function.
|
|
48
|
+
self.assertEqual(c.inputs, set(['input_1', 'input_2']))
|
|
49
|
+
# The default values of the inputs are stored in the defaults member.
|
|
50
|
+
self.assertEqual(c.defaults, {'input_2': None})
|
|
51
|
+
# The outputs are a tuple of the output names. For a calc without
|
|
52
|
+
# explicitly listed outputs has only one output, its name.
|
|
53
|
+
self.assertEqual(c.outputs, ('result',))
|
|
54
|
+
# The input documentation is stored in the input_docs member.
|
|
55
|
+
self.assertIn('input_1', c.input_docs)
|
|
56
|
+
self.assertIn('input_2', c.input_docs)
|
|
57
|
+
self.assertEqual(len(c.input_docs), 2)
|
|
58
|
+
self.assertIn('The number of elements to include in the result.',
|
|
59
|
+
c.input_docs['input_1'])
|
|
60
|
+
self.assertIn('The object to put in the list.',
|
|
61
|
+
c.input_docs['input_2'])
|
|
62
|
+
# The output documentation is stored in the output_docs member.
|
|
63
|
+
self.assertIn('result', c.output_docs)
|
|
64
|
+
self.assertIn('A list of `input_1` occurrences of `input_2`.',
|
|
65
|
+
c.output_docs['result'])
|
|
66
|
+
self.assertEqual(len(c.output_docs), 1)
|
|
67
|
+
# The calculation can be called using its normal signature.
|
|
68
|
+
self.assertEqual(c(1), {'result': [None]})
|
|
69
|
+
self.assertEqual(c(2, 0), {'result': [0, 0]})
|
|
70
|
+
# The call method is basically an alias for the __call__ method.
|
|
71
|
+
self.assertEqual(c.call(1), {'result': [None]})
|
|
72
|
+
self.assertEqual(c.call(2, 0), {'result': [0, 0]})
|
|
73
|
+
# The call can also be forced to be either eager or lazy--when lazy,
|
|
74
|
+
# the return value is a lazy dict, and the calc isn't actually run until
|
|
75
|
+
# the values are requested; when eager, the call is run right away, and
|
|
76
|
+
# the return value is an frozendict instead of a lazydict.
|
|
77
|
+
self.assertIsInstance(c.eager_call(1), pdict)
|
|
78
|
+
self.assertEqual(c.eager_call(1), {'result': [None]})
|
|
79
|
+
self.assertEqual(c.eager_call(2, 0), {'result': [0, 0]})
|
|
80
|
+
self.assertIsInstance(c.lazy_call(1), ldict)
|
|
81
|
+
self.assertEqual(c.lazy_call(1), {'result': [None]})
|
|
82
|
+
self.assertEqual(c.lazy_call(2, 0), {'result': [0, 0]})
|
|
83
|
+
# It can also be called using the mapcall method.
|
|
84
|
+
m1 = dict(input_1=1)
|
|
85
|
+
m2 = dict(input_1=2, input_2=0)
|
|
86
|
+
self.assertEqual(c.mapcall(m1), {'result': [None]})
|
|
87
|
+
self.assertEqual(c.mapcall(m2), {'result': [0,0]})
|
|
88
|
+
# These can also be lazy or eager.
|
|
89
|
+
self.assertIsInstance(c.eager_mapcall(m1), pdict)
|
|
90
|
+
self.assertEqual(c.eager_mapcall(m1), {'result': [None]})
|
|
91
|
+
self.assertEqual(c.eager_mapcall(m2), {'result': [0, 0]})
|
|
92
|
+
self.assertIsInstance(c.lazy_mapcall(m1), ldict)
|
|
93
|
+
self.assertEqual(c.lazy_mapcall(m1), {'result': [None]})
|
|
94
|
+
self.assertEqual(c.lazy_mapcall(m2), {'result': [0, 0]})
|
|
95
|
+
# Calculations can have multiple outputs as well as multiple inputs.
|
|
96
|
+
@calc('out1', 'out2', 'out3')
|
|
97
|
+
def sample_calc(in1, in2, in3):
|
|
98
|
+
return (in1 + 1, in2 + 2, in3 + 3)
|
|
99
|
+
res = sample_calc.calc(1, 2, 3)
|
|
100
|
+
self.assertIsInstance(res, ldict)
|
|
101
|
+
self.assertEqual(len(res), 3)
|
|
102
|
+
self.assertEqual(res['out1'], 2)
|
|
103
|
+
self.assertEqual(res['out2'], 4)
|
|
104
|
+
self.assertEqual(res['out3'], 6)
|
|
105
|
+
# New calcs can be made that change the names of the calculation
|
|
106
|
+
# variables (the inputs and outputs) using the tr (translate) method.
|
|
107
|
+
sample_tr = sample_calc.calc.rename_keys(out1='x', out2='y', in3='z')
|
|
108
|
+
self.assertEqual(sample_tr.outputs, ('x', 'y', 'out3'))
|
|
109
|
+
self.assertEqual(len(sample_tr.inputs), 3)
|
|
110
|
+
self.assertIn('in1', sample_tr.inputs)
|
|
111
|
+
self.assertIn('in2', sample_tr.inputs)
|
|
112
|
+
self.assertIn('z', sample_tr.inputs)
|
|
113
|
+
res = sample_tr.mapcall({'in1':1, 'in2':2, 'z':3})
|
|
114
|
+
self.assertIsInstance(res, ldict)
|
|
115
|
+
self.assertEqual(len(res), 3)
|
|
116
|
+
self.assertEqual(res['x'], 2)
|
|
117
|
+
self.assertEqual(res['y'], 4)
|
|
118
|
+
self.assertEqual(res['out3'], 6)
|
|
119
|
+
def test_is_calc(self):
|
|
120
|
+
from immlib.workflow import (calc, is_calcfn)
|
|
121
|
+
@calc
|
|
122
|
+
def result(input_1, input_2=None):
|
|
123
|
+
return ([input_2] * input_1,)
|
|
124
|
+
# is_calc(x) is just an alias for isinstance(x, calc).
|
|
125
|
+
self.assertTrue(is_calcfn(result))
|
|
126
|
+
self.assertFalse(is_calcfn(lambda x:x))
|
|
127
|
+
self.assertEqual(result.calc(2,0)['result'], [0, 0])
|
|
128
|
+
def test_plan(self):
|
|
129
|
+
import numpy as np
|
|
130
|
+
from immlib.workflow import (calc, plan, plandict)
|
|
131
|
+
# Plans are just collections of calc objects, each of which gets built
|
|
132
|
+
# into a directed acyclic graph of calculation dependencies.
|
|
133
|
+
@calc('weights', lazy=False)
|
|
134
|
+
def normal_pdf(x, mu=0, std=1):
|
|
135
|
+
"""Calculates the probability densities for a normal distribution.
|
|
136
|
+
|
|
137
|
+
Inputs
|
|
138
|
+
------
|
|
139
|
+
x : array-like
|
|
140
|
+
The input values at which to calculate the normal PDF.
|
|
141
|
+
mu : number, optional
|
|
142
|
+
The mean of the normal distribution; the default is 0.
|
|
143
|
+
std : number, optional
|
|
144
|
+
The standard deviation of the distribution; the default is 1.
|
|
145
|
+
|
|
146
|
+
Outputs
|
|
147
|
+
-------
|
|
148
|
+
weights : array-like
|
|
149
|
+
The probability densities of the normal distribution at the
|
|
150
|
+
given set of values in `x`.
|
|
151
|
+
"""
|
|
152
|
+
w = np.exp(-0.5 * ((x - mu)/std)**2) / (np.sqrt(2*np.pi) * std)
|
|
153
|
+
return (w,)
|
|
154
|
+
@calc('mean')
|
|
155
|
+
def weighted_mean(x, weights):
|
|
156
|
+
"""Calculates the weighted mean.
|
|
157
|
+
|
|
158
|
+
Inputs
|
|
159
|
+
------
|
|
160
|
+
x : array-like
|
|
161
|
+
The values to be averaged.
|
|
162
|
+
weights : array-like
|
|
163
|
+
The weights of the values in `x`.
|
|
164
|
+
|
|
165
|
+
Outputs
|
|
166
|
+
-------
|
|
167
|
+
mean : number
|
|
168
|
+
The weighted mean of the inputs.
|
|
169
|
+
"""
|
|
170
|
+
mean = np.sum(x * weights) / np.sum(weights)
|
|
171
|
+
return (mean,)
|
|
172
|
+
# Filter calculations can be used to update the input variables to a
|
|
173
|
+
# plan--they are calc units that accept only 1 input and that return
|
|
174
|
+
# same input.
|
|
175
|
+
@calc('x')
|
|
176
|
+
def filter_x(x):
|
|
177
|
+
x = np.asarray(x)
|
|
178
|
+
assert len(x.shape) == 1, "x must be a vector"
|
|
179
|
+
assert np.issubdtype(x.dtype, np.number), "x must be numeric"
|
|
180
|
+
return (x,)
|
|
181
|
+
# The calculations are given names (keys) and put together in a plan.
|
|
182
|
+
nwm = plan(
|
|
183
|
+
weights_step=normal_pdf,
|
|
184
|
+
mean_step=weighted_mean,
|
|
185
|
+
filter_x=filter_x)
|
|
186
|
+
# This creates a plan object, which stores these computations.
|
|
187
|
+
self.assertIsInstance(nwm, plan)
|
|
188
|
+
# The plan keeps track lots of meta-data, including an agglomeration of
|
|
189
|
+
# the meta-data of its calculations.
|
|
190
|
+
self.assertEqual(nwm.inputs, set(['x', 'mu', 'std']))
|
|
191
|
+
self.assertEqual(nwm.outputs, set(['weights', 'mean']))
|
|
192
|
+
self.assertEqual(nwm.defaults, {'mu': 0, 'std': 1})
|
|
193
|
+
# We can provide a plan with its parameters in order to create a
|
|
194
|
+
# plandict, which is a lazydict that agglomerates all of the input and
|
|
195
|
+
# output values of all the calculations.
|
|
196
|
+
pd = nwm(x=[-1.0, 1.0, 2.0, 8.5], mu=1.5)
|
|
197
|
+
self.assertIsInstance(pd, plandict)
|
|
198
|
+
self.assertEqual(len(pd), 5)
|
|
199
|
+
# In this case, because we have a non-lazy calc (normal_pdf), all of
|
|
200
|
+
# that calc's inputs are also automatically ready (this is not a
|
|
201
|
+
# surprise--its other inputs are plain params so are not lazy objects).
|
|
202
|
+
self.assertFalse(pd.is_lazy('mu'))
|
|
203
|
+
self.assertFalse(pd.is_lazy('std'))
|
|
204
|
+
self.assertTrue(pd.is_ready('mu'))
|
|
205
|
+
self.assertTrue(pd.is_ready('std'))
|
|
206
|
+
# The weights outputs should be ready because it was declared to be
|
|
207
|
+
# non-lazy; the mean should remain lazy, though.
|
|
208
|
+
self.assertTrue(pd.is_lazy('weights'))
|
|
209
|
+
self.assertFalse(pd.is_ready('weights'))
|
|
210
|
+
self.assertTrue(pd.is_lazy('mean'))
|
|
211
|
+
self.assertFalse(pd.is_ready('mean'))
|
|
212
|
+
# It will have converted the x value into an array.
|
|
213
|
+
self.assertIsInstance(pd['x'], np.ndarray)
|
|
214
|
+
self.assertTrue(np.array_equal(pd['x'], [-1, 1, 2, 8.5]))
|
|
215
|
+
self.assertEqual(pd['mu'], 1.5)
|
|
216
|
+
self.assertEqual(pd['std'], 1)
|
|
217
|
+
self.assertAlmostEqual(pd['mean'], 1.4392777559)
|
|
218
|
+
# We can update the plandict by making a new one.
|
|
219
|
+
pd2 = plandict(pd, x=[0, 1, 2, 8.5])
|
|
220
|
+
self.assertIsInstance(pd2['x'], np.ndarray)
|
|
221
|
+
self.assertTrue(np.array_equal(pd2['x'], [0, 1, 2, 8.5]))
|
|
222
|
+
self.assertEqual(pd2['mu'], 1.5)
|
|
223
|
+
self.assertEqual(pd2['std'], 1)
|
|
224
|
+
self.assertAlmostEqual(pd2['mean'], 1.266956394834)
|
|
225
|
+
pd2 = plandict(pd, mu=2.5)
|
|
226
|
+
self.assertIsInstance(pd2['x'], np.ndarray)
|
|
227
|
+
self.assertTrue(np.array_equal(pd2['x'], [-1, 1, 2, 8.5]))
|
|
228
|
+
self.assertEqual(pd2['mu'], 2.5)
|
|
229
|
+
self.assertEqual(pd2['std'], 1)
|
|
230
|
+
self.assertAlmostEqual(pd2['mean'], 1.726118628968)
|
|
231
|
+
# We can also make a transient plandict...
|
|
232
|
+
tpd = pd.transient()
|
|
233
|
+
tpd['x'] = [0, 1, 2, 8.5]
|
|
234
|
+
self.assertIsInstance(tpd['x'], np.ndarray)
|
|
235
|
+
self.assertTrue(np.array_equal(tpd['x'], [0, 1, 2, 8.5]))
|
|
236
|
+
self.assertEqual(tpd['mu'], 1.5)
|
|
237
|
+
self.assertEqual(tpd['std'], 1)
|
|
238
|
+
self.assertAlmostEqual(tpd['mean'], 1.266956394834)
|
|
239
|
+
tpd = pd.transient()
|
|
240
|
+
tpd['mu'] = 2.5
|
|
241
|
+
self.assertIsInstance(tpd['x'], np.ndarray)
|
|
242
|
+
self.assertTrue(np.array_equal(tpd['x'], [-1, 1, 2, 8.5]))
|
|
243
|
+
self.assertEqual(tpd['mu'], 2.5)
|
|
244
|
+
self.assertEqual(tpd['std'], 1)
|
|
245
|
+
self.assertAlmostEqual(tpd['mean'], 1.726118628968)
|
|
246
|
+
# Since we marked the filter as non-lazy, it should raise errors when
|
|
247
|
+
# the plan is fulfilled.
|
|
248
|
+
with self.assertRaises(RuntimeError): nwm(x=10)
|
|
249
|
+
# We should also make sure the documentation is getting loaded
|
|
250
|
+
# correctly.
|
|
251
|
+
for k in ('x', 'mu', 'std'):
|
|
252
|
+
self.assertIn(k, nwm.inputs)
|
|
253
|
+
self.assertIn(k, nwm.input_docs)
|
|
254
|
+
for k in ('mean', 'weights'):
|
|
255
|
+
self.assertIn(k, nwm.output_docs)
|
|
256
|
+
def test_multifilter(self):
|
|
257
|
+
"""Tests the ability of plans to contain multi-input filters."""
|
|
258
|
+
import numpy as np
|
|
259
|
+
from immlib.workflow import calc, plan
|
|
260
|
+
@calc('a', 'b', 'c', lazy=False)
|
|
261
|
+
def filter_bccoords(a=None, b=None, c=None):
|
|
262
|
+
n_given = 3 - (int(a is None) + int(b is None) + int(c is None))
|
|
263
|
+
if n_given < 2:
|
|
264
|
+
raise ValueError("at least two of a, b, and c must be provided")
|
|
265
|
+
elif n_given == 2:
|
|
266
|
+
if a is None:
|
|
267
|
+
a = 1 - (b + c)
|
|
268
|
+
elif b is None:
|
|
269
|
+
b = 1 - (a + c)
|
|
270
|
+
elif c is None:
|
|
271
|
+
c = 1 - (a + b)
|
|
272
|
+
return (a, b, c)
|
|
273
|
+
@calc('a_coords', 'b_coords', 'c_coords', lazy=False)
|
|
274
|
+
def filter_tricoords(a_coords, b_coords, c_coords):
|
|
275
|
+
a_coords = np.array(a_coords)
|
|
276
|
+
b_coords = np.array(b_coords)
|
|
277
|
+
c_coords = np.array(c_coords)
|
|
278
|
+
a_coords.flags.writeable = False
|
|
279
|
+
b_coords.flags.writeable = False
|
|
280
|
+
c_coords.flags.writeable = False
|
|
281
|
+
return (a_coords, b_coords, c_coords)
|
|
282
|
+
@calc('coords')
|
|
283
|
+
def calc_coords(a_coords, b_coords, c_coords, a, b, c):
|
|
284
|
+
return (a*a_coords + b*b_coords + c*c_coords,)
|
|
285
|
+
p = plan(
|
|
286
|
+
bcfilter=filter_bccoords,
|
|
287
|
+
trifilter=filter_tricoords,
|
|
288
|
+
coords=calc_coords)
|
|
289
|
+
# The main thing is that this plan should not have any trouble filling
|
|
290
|
+
# in the three values.
|
|
291
|
+
tri = {'a_coords': (0,0), 'b_coords':(0,1), 'c_coords': (1,0)}
|
|
292
|
+
u = p(a=0.25, b=0.25, **tri)
|
|
293
|
+
self.assertEqual(u['c'], 0.5)
|
|
294
|
+
u = p(a=0.25, c=0.25, **tri)
|
|
295
|
+
self.assertEqual(u['b'], 0.5)
|
|
296
|
+
u = p(c=0.25, b=0.25, **tri)
|
|
297
|
+
self.assertEqual(u['a'], 0.5)
|
|
298
|
+
def test_pathcache(self):
|
|
299
|
+
"""Tests that the pathcache argument works correctly."""
|
|
300
|
+
# We make a temporary cache path directory for all of this:
|
|
301
|
+
from tempfile import TemporaryDirectory
|
|
302
|
+
from joblib import Memory
|
|
303
|
+
from immlib import calc, plan
|
|
304
|
+
from immlib.workflow import to_calc
|
|
305
|
+
with TemporaryDirectory() as tmpdir:
|
|
306
|
+
self.pc_runcount = 0
|
|
307
|
+
@calc('outputval1', 'outputval2', pathcache=tmpdir)
|
|
308
|
+
def test_cache1(inputval1, inputval2):
|
|
309
|
+
self.pc_runcount = self.pc_runcount + 1
|
|
310
|
+
return (inputval1 // inputval2, inputval1 % inputval2)
|
|
311
|
+
d = test_cache1.calc(10, 3)
|
|
312
|
+
self.assertEqual(d['outputval1'], 3)
|
|
313
|
+
self.assertEqual(d['outputval2'], 1)
|
|
314
|
+
self.assertEqual(self.pc_runcount, 1)
|
|
315
|
+
d = test_cache1.calc(10, 3)
|
|
316
|
+
self.assertEqual(d['outputval1'], 3)
|
|
317
|
+
self.assertEqual(d['outputval2'], 1)
|
|
318
|
+
self.assertEqual(self.pc_runcount, 1)
|
|
319
|
+
d = test_cache1.calc(10, 5)
|
|
320
|
+
self.assertEqual(d['outputval1'], 2)
|
|
321
|
+
self.assertEqual(d['outputval2'], 0)
|
|
322
|
+
self.assertEqual(self.pc_runcount, 2)
|
|
323
|
+
d = test_cache1.calc(10, 5)
|
|
324
|
+
self.assertEqual(d['outputval1'], 2)
|
|
325
|
+
self.assertEqual(d['outputval2'], 0)
|
|
326
|
+
self.assertEqual(self.pc_runcount, 2)
|
|
327
|
+
# We can also use pathcache=True and pass the tmpdir as a cache_path
|
|
328
|
+
# parameter.
|
|
329
|
+
self.pc_runcount = 0
|
|
330
|
+
@calc('outputval1', 'outputval2', pathcache=True)
|
|
331
|
+
def test_cache2(inputval1, inputval2):
|
|
332
|
+
self.pc_runcount = self.pc_runcount + 1
|
|
333
|
+
return (inputval1 // inputval2, inputval1 % inputval2)
|
|
334
|
+
# The pathcache gets noted
|
|
335
|
+
c = to_calc(test_cache2)
|
|
336
|
+
self.assertTrue(c.pathcache)
|
|
337
|
+
# Now make a plan.
|
|
338
|
+
p = plan(test=test_cache2)
|
|
339
|
+
# No cache_path, no caching.
|
|
340
|
+
d = p(inputval1=10, inputval2=3)
|
|
341
|
+
self.assertEqual(d['cache_path'], None)
|
|
342
|
+
self.assertEqual(d['outputval1'], 3)
|
|
343
|
+
self.assertEqual(d['outputval2'], 1)
|
|
344
|
+
self.assertEqual(self.pc_runcount, 1)
|
|
345
|
+
d = p(inputval1=10, inputval2=3)
|
|
346
|
+
self.assertEqual(d['cache_path'], None)
|
|
347
|
+
self.assertEqual(d['outputval1'], 3)
|
|
348
|
+
self.assertEqual(d['outputval2'], 1)
|
|
349
|
+
self.assertEqual(self.pc_runcount, 2)
|
|
350
|
+
# With a cache_path, it gets cached.
|
|
351
|
+
d = p(inputval1=10, inputval2=3, cache_path=tmpdir)
|
|
352
|
+
self.assertEqual(d['cache_path'], tmpdir)
|
|
353
|
+
self.assertEqual(d['outputval1'], 3)
|
|
354
|
+
self.assertEqual(d['outputval2'], 1)
|
|
355
|
+
self.assertEqual(self.pc_runcount, 3)
|
|
356
|
+
d = p(inputval1=10, inputval2=3, cache_path=tmpdir)
|
|
357
|
+
self.assertEqual(d['cache_path'], tmpdir)
|
|
358
|
+
self.assertEqual(d['outputval1'], 3)
|
|
359
|
+
self.assertEqual(d['outputval2'], 1)
|
|
360
|
+
self.assertEqual(self.pc_runcount, 3)
|
|
361
|
+
# We can also test the version of this where we include cache_path
|
|
362
|
+
# as an input parameter.
|
|
363
|
+
self.pc_runcount = 0
|
|
364
|
+
@calc('outputval1', 'outputval2', 'out_cpath', pathcache=True)
|
|
365
|
+
def test_cache3(inputval1, inputval2, cache_path=None):
|
|
366
|
+
self.pc_runcount = self.pc_runcount + 1
|
|
367
|
+
return (inputval1 // inputval2, inputval1 % inputval2,
|
|
368
|
+
cache_path)
|
|
369
|
+
p = plan(test=test_cache3)
|
|
370
|
+
# No cache_path, no caching.
|
|
371
|
+
d = p(inputval1=10, inputval2=3)
|
|
372
|
+
self.assertEqual(d['outputval1'], 3)
|
|
373
|
+
self.assertEqual(d['outputval2'], 1)
|
|
374
|
+
self.assertEqual(d['out_cpath'], None)
|
|
375
|
+
self.assertEqual(self.pc_runcount, 1)
|
|
376
|
+
d = p(inputval1=10, inputval2=3)
|
|
377
|
+
self.assertEqual(d['outputval1'], 3)
|
|
378
|
+
self.assertEqual(d['outputval2'], 1)
|
|
379
|
+
self.assertEqual(d['out_cpath'], None)
|
|
380
|
+
self.assertEqual(self.pc_runcount, 2)
|
|
381
|
+
# With a cache_path, it gets cached.
|
|
382
|
+
d = p(inputval1=10, inputval2=3, cache_path=tmpdir)
|
|
383
|
+
self.assertEqual(d['outputval1'], 3)
|
|
384
|
+
self.assertEqual(d['outputval2'], 1)
|
|
385
|
+
self.assertEqual(d['out_cpath'], tmpdir)
|
|
386
|
+
self.assertEqual(self.pc_runcount, 3)
|
|
387
|
+
d = p(inputval1=10, inputval2=3, cache_path=tmpdir)
|
|
388
|
+
self.assertEqual(d['outputval1'], 3)
|
|
389
|
+
self.assertEqual(d['outputval2'], 1)
|
|
390
|
+
self.assertEqual(d['out_cpath'], tmpdir)
|
|
391
|
+
self.assertEqual(self.pc_runcount, 3)
|
|
392
|
+
def test_decstack(self):
|
|
393
|
+
"Tests the ability to stack calc decorations with other decorators."
|
|
394
|
+
from immlib.workflow import calc, plan
|
|
395
|
+
from immlib.util import tensor_args
|
|
396
|
+
import numpy as np, torch
|
|
397
|
+
# Create a calculation that computes a normalized vector `u` and a
|
|
398
|
+
# length `xlen` given an unnormalized vector `x`.
|
|
399
|
+
@tensor_args(keep_arrays=True)
|
|
400
|
+
@calc('u', 'xlen')
|
|
401
|
+
def normalize_vector(x):
|
|
402
|
+
xlen = torch.sqrt(torch.sum(x**2))
|
|
403
|
+
u = x / xlen
|
|
404
|
+
return (u, xlen)
|
|
405
|
+
# Create another calculation that finds the signed distance between a
|
|
406
|
+
# point `y` and the vector `x`, as well as the point of intersection.
|
|
407
|
+
@calc('distance', 'intersection')
|
|
408
|
+
@tensor_args(keep_arrays=True)
|
|
409
|
+
def point_vec_intersection(u, y):
|
|
410
|
+
d = torch.dot(u, y)
|
|
411
|
+
return (d, u*d)
|
|
412
|
+
p = plan(step1=normalize_vector, step2=point_vec_intersection)
|
|
413
|
+
pd = p(x=[0.0, 1.0], y=[1.0, 1.0])
|
|
414
|
+
self.assertEqual(pd['distance'], 1.0)
|
|
415
|
+
self.assertIsInstance(pd['u'], np.ndarray)
|
|
416
|
+
def test_tplandict(self):
|
|
417
|
+
"Tests the tplandict type."
|
|
418
|
+
pass
|
|
@@ -0,0 +1,248 @@
|
|
|
1
|
+
# -*- coding: utf-8 -*-
|
|
2
|
+
################################################################################
|
|
3
|
+
# immlib/test/workflow/test_plantype.py
|
|
4
|
+
#
|
|
5
|
+
# Tests of the plantype system in immlib: i.e., tests for the code in the
|
|
6
|
+
# immlib.workflow._plantype module.
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
# Dependencies #################################################################
|
|
10
|
+
|
|
11
|
+
from unittest import TestCase
|
|
12
|
+
import numpy as np
|
|
13
|
+
|
|
14
|
+
from ...workflow import (planobject, calc)
|
|
15
|
+
|
|
16
|
+
# The plantype type can be used as a metaclass for a class in order to
|
|
17
|
+
# make that class into a workflow/plantype class. Alternately, we can
|
|
18
|
+
# just inherit from planobject.
|
|
19
|
+
class TriangleData(planobject):
|
|
20
|
+
def __init__(self, a, b, c):
|
|
21
|
+
self.a = a
|
|
22
|
+
self.b = b
|
|
23
|
+
self.c = c
|
|
24
|
+
@calc('a', lazy=False)
|
|
25
|
+
def filter_a(a):
|
|
26
|
+
a = np.array(a)
|
|
27
|
+
assert a.shape == (2,)
|
|
28
|
+
return a
|
|
29
|
+
@calc('b', lazy=False)
|
|
30
|
+
def filter_b(b):
|
|
31
|
+
b = np.array(b)
|
|
32
|
+
assert b.shape == (2,)
|
|
33
|
+
return b
|
|
34
|
+
@calc('c', lazy=False)
|
|
35
|
+
def filter_c(c):
|
|
36
|
+
c = np.array(c)
|
|
37
|
+
assert c.shape == (2,)
|
|
38
|
+
return c
|
|
39
|
+
@calc('base', 'height')
|
|
40
|
+
def calc_triangle_base(a, b, c):
|
|
41
|
+
'''calc_triangle_base computes the base (x-width) of the
|
|
42
|
+
triangle a-b-c.
|
|
43
|
+
|
|
44
|
+
Inputs
|
|
45
|
+
------
|
|
46
|
+
a : list-like
|
|
47
|
+
The (x,y) coordinate of point a in triangle a-b-c.
|
|
48
|
+
b : list-like
|
|
49
|
+
The (x,y) coordinate of point b in triangle a-b-c.
|
|
50
|
+
c : list-like
|
|
51
|
+
The (x,y) coordinate of point c in triangle a-b-c.
|
|
52
|
+
|
|
53
|
+
Outputs
|
|
54
|
+
-------
|
|
55
|
+
base : number
|
|
56
|
+
The base, or width, of the triangle a-b-c.
|
|
57
|
+
height : number
|
|
58
|
+
The height of the triangle a-b-c.
|
|
59
|
+
'''
|
|
60
|
+
print('Calculating base...')
|
|
61
|
+
# We pick AB as the base.
|
|
62
|
+
base = b - a
|
|
63
|
+
baselen = np.hypot(base[0], base[1])
|
|
64
|
+
ubase = base / baselen
|
|
65
|
+
print('Calculating height...')
|
|
66
|
+
uortho = np.array([[0,-1],[1,0]]) @ ubase
|
|
67
|
+
height = np.abs(np.dot(uortho, c - a))
|
|
68
|
+
return (baselen, height)
|
|
69
|
+
@calc('area')
|
|
70
|
+
def calc_triangle_area(base, height):
|
|
71
|
+
'''calc_triangle_are computes the area of a triangle with a
|
|
72
|
+
given base and height.
|
|
73
|
+
|
|
74
|
+
Inputs
|
|
75
|
+
------
|
|
76
|
+
base : number
|
|
77
|
+
The base of the triangle.
|
|
78
|
+
height : number
|
|
79
|
+
The height of the triangle.
|
|
80
|
+
|
|
81
|
+
Outputs
|
|
82
|
+
-------
|
|
83
|
+
area : number
|
|
84
|
+
The area of the triangle with the given base and height.
|
|
85
|
+
'''
|
|
86
|
+
print('Calculating area...')
|
|
87
|
+
return {'area': base * height * 0.5}
|
|
88
|
+
def __eq__(self, other):
|
|
89
|
+
if not isinstance(other, TriangleData):
|
|
90
|
+
return False
|
|
91
|
+
return (
|
|
92
|
+
np.array_equal(self.a, other.a) and
|
|
93
|
+
np.array_equal(self.b, other.b) and
|
|
94
|
+
np.array_equal(self.c, other.c))
|
|
95
|
+
def __hash__(self):
|
|
96
|
+
return super().__hash__()
|
|
97
|
+
|
|
98
|
+
class TestWorkflowPlanType(TestCase):
|
|
99
|
+
"""Tests the immlib.workflow._plantype module."""
|
|
100
|
+
def test_plantype(self):
|
|
101
|
+
import numpy as np
|
|
102
|
+
from immlib.workflow import (
|
|
103
|
+
planobject, plantype, calc, is_plantype, is_planobject)
|
|
104
|
+
import sys, io, pickle
|
|
105
|
+
# We can instantiate the plantype as normal:
|
|
106
|
+
tri = TriangleData((0,0), (1,0), (0,1))
|
|
107
|
+
# We've setup the class to print some messages the first time values get
|
|
108
|
+
# calculated, so we capture them here.
|
|
109
|
+
sys.stdout = io.StringIO()
|
|
110
|
+
self.assertEqual(tri.base, 1)
|
|
111
|
+
self.assertEqual(sys.stdout.getvalue(),
|
|
112
|
+
'Calculating base...\nCalculating height...\n')
|
|
113
|
+
self.assertEqual(tri.height, 1)
|
|
114
|
+
self.assertEqual(sys.stdout.getvalue(),
|
|
115
|
+
'Calculating base...\nCalculating height...\n')
|
|
116
|
+
sys.stdout = io.StringIO()
|
|
117
|
+
self.assertEqual(tri.area, 0.5)
|
|
118
|
+
self.assertEqual(sys.stdout.getvalue(), 'Calculating area...\n')
|
|
119
|
+
# Objects of type planobject are immutable:
|
|
120
|
+
with self.assertRaises(TypeError):
|
|
121
|
+
tri.a = (2,2)
|
|
122
|
+
with self.assertRaises(TypeError):
|
|
123
|
+
del tri.a
|
|
124
|
+
# Test some basic attribute stuff...
|
|
125
|
+
with self.assertRaises(AttributeError):
|
|
126
|
+
tri.notanattribute
|
|
127
|
+
# The dir function should return normal object items as well as the
|
|
128
|
+
# plan's values:
|
|
129
|
+
d = dir(tri)
|
|
130
|
+
for k in ('a', 'b', 'c', 'base', 'height', 'area', '__class__',
|
|
131
|
+
'_plandict_', '__new__'):
|
|
132
|
+
self.assertIn(k, d)
|
|
133
|
+
# Trying to initialize an object after it has been initialized should
|
|
134
|
+
# raise an error.
|
|
135
|
+
with self.assertRaises(RuntimeError):
|
|
136
|
+
tri.__init__((0,0), (1,0), (0,1))
|
|
137
|
+
# Init methods can be called in parent classes without problems.
|
|
138
|
+
class ChildTri(TriangleData):
|
|
139
|
+
def __init__(self, a, b, c):
|
|
140
|
+
super().__init__(a, b, c)
|
|
141
|
+
a = np.asarray(a)
|
|
142
|
+
self.a = (0,0)
|
|
143
|
+
self.b = b - a
|
|
144
|
+
self.c = c - a
|
|
145
|
+
tri = ChildTri((0,1), (1,0), (0,2))
|
|
146
|
+
self.assertTrue(np.array_equal(tri.a, (0,0)))
|
|
147
|
+
self.assertTrue(np.array_equal(tri.b, (1,-1)))
|
|
148
|
+
self.assertTrue(np.array_equal(tri.c, (0,1)))
|
|
149
|
+
# If we try to make a planobject that sets a non-input in its __init__
|
|
150
|
+
# method, it should raise an error.
|
|
151
|
+
class BadTriData(TriangleData):
|
|
152
|
+
def __init__(self, a, b, c):
|
|
153
|
+
super().__init__(a, b, c)
|
|
154
|
+
self.base = 10
|
|
155
|
+
with self.assertRaises(ValueError):
|
|
156
|
+
tri = BadTriData((0,0), (1,0), (0,1))
|
|
157
|
+
# There is an automatically defined init function.
|
|
158
|
+
class AutoInit(planobject):
|
|
159
|
+
@calc('x')
|
|
160
|
+
def filter_x(x):
|
|
161
|
+
return (float(x),)
|
|
162
|
+
@calc('y')
|
|
163
|
+
def filter_y(y):
|
|
164
|
+
return (float(y),)
|
|
165
|
+
@calc('z')
|
|
166
|
+
def filter_z(z):
|
|
167
|
+
return (float(z),)
|
|
168
|
+
@calc('f')
|
|
169
|
+
def calc_outputs(x, y, z=0):
|
|
170
|
+
result = x + y*z
|
|
171
|
+
return (result,)
|
|
172
|
+
ai = AutoInit(x=1, y=2, z='4')
|
|
173
|
+
self.assertIsInstance(ai, AutoInit)
|
|
174
|
+
self.assertEqual(ai.x, 1)
|
|
175
|
+
self.assertEqual(ai.y, 2)
|
|
176
|
+
self.assertEqual(ai.z, 4)
|
|
177
|
+
self.assertEqual(ai.f, 9)
|
|
178
|
+
self.assertIsInstance(ai.x, float)
|
|
179
|
+
self.assertIsInstance(ai.y, float)
|
|
180
|
+
self.assertIsInstance(ai.z, float)
|
|
181
|
+
self.assertIsInstance(ai.f, float)
|
|
182
|
+
# If we don't provide an object with all its inputs, there's an error.
|
|
183
|
+
with self.assertRaises(ValueError):
|
|
184
|
+
ai = AutoInit(x=1, z='4')
|
|
185
|
+
# planobjects can be serialized and deserialized:
|
|
186
|
+
tri = TriangleData((0,0), (1,0), (0,1))
|
|
187
|
+
self.assertEqual(tri, pickle.loads(pickle.dumps(tri)))
|
|
188
|
+
# planobjects are by default only equal on the basis of their type and
|
|
189
|
+
# their inputs.
|
|
190
|
+
tri1 = TriangleData((0,0), (1,0), (0,1))
|
|
191
|
+
tri2 = TriangleData((0.0,0.0), (1.0,0.0), (0.0,1.0))
|
|
192
|
+
tri3 = TriangleData((0.1,0.0), (1.0,0.0), (0.0,1.0))
|
|
193
|
+
tri4 = ChildTri((0,0), (1,0), (0,1))
|
|
194
|
+
self.assertEqual(tri1, tri2)
|
|
195
|
+
self.assertNotEqual(tri2, tri3)
|
|
196
|
+
self.assertNotEqual(tri1, tri4)
|
|
197
|
+
self.assertEqual(hash(tri1), hash(tri2))
|
|
198
|
+
# A planobject can be made transient and a transient one can be made
|
|
199
|
+
# persistent again.
|
|
200
|
+
ttri1 = tri1.transient()
|
|
201
|
+
self.assertEqual(tri1, ttri1)
|
|
202
|
+
self.assertTrue(tri1.is_persistent())
|
|
203
|
+
self.assertFalse(ttri1.is_persistent())
|
|
204
|
+
self.assertTrue(np.array_equal(ttri1.a, (0, 0)))
|
|
205
|
+
self.assertEqual(ttri1.area, 0.5)
|
|
206
|
+
ttri1.a = (0.2, 0.2)
|
|
207
|
+
self.assertTrue(np.array_equal(ttri1.a, (0.2, 0.2)))
|
|
208
|
+
self.assertAlmostEqual(ttri1.area, 0.3)
|
|
209
|
+
tri5 = ttri1.persistent()
|
|
210
|
+
self.assertEqual(ttri1, tri5)
|
|
211
|
+
self.assertTrue(tri5.is_persistent())
|
|
212
|
+
self.assertTrue(np.array_equal(tri5.a, (0.2, 0.2)))
|
|
213
|
+
self.assertAlmostEqual(tri5.area, 0.3)
|
|
214
|
+
# At this point, we can return the normal standard output (we should be
|
|
215
|
+
# done printing things).
|
|
216
|
+
sys.stdout = sys.__stdout__
|
|
217
|
+
# We can turn planobjects into strings:
|
|
218
|
+
class SimpleObj(planobject):
|
|
219
|
+
@calc('x')
|
|
220
|
+
def filter_x(x):
|
|
221
|
+
return (int(x),)
|
|
222
|
+
@calc('z')
|
|
223
|
+
def calc_z(x, y):
|
|
224
|
+
return ((x*y),)
|
|
225
|
+
obj = SimpleObj(x=10.0, y=2.0)
|
|
226
|
+
self.assertEqual(str(obj), 'SimpleObj(x=<lazy>, y=2.0; z=<lazy>)')
|
|
227
|
+
self.assertTrue(
|
|
228
|
+
repr(obj).startswith(
|
|
229
|
+
f'{__name__}.SimpleObj(x=10, y=2.0; z=lazy(<'))
|
|
230
|
+
self.assertTrue(
|
|
231
|
+
repr(obj).endswith(
|
|
232
|
+
'>: waiting))'))
|
|
233
|
+
self.assertIsInstance(obj.x, int)
|
|
234
|
+
self.assertEqual(str(obj), 'SimpleObj(x=10, y=2.0; z=<lazy>)')
|
|
235
|
+
# If the class has no outputs, there is no semicolon.
|
|
236
|
+
class TrivialObj(planobject):
|
|
237
|
+
@calc('x')
|
|
238
|
+
def filter_x(x):
|
|
239
|
+
return (int(x),)
|
|
240
|
+
trivobj = TrivialObj(x=10.0)
|
|
241
|
+
self.assertEqual(str(trivobj), 'TrivialObj(x=<lazy>)')
|
|
242
|
+
self.assertEqual(trivobj.x, 10)
|
|
243
|
+
self.assertEqual(repr(trivobj), f'{__name__}.TrivialObj(x=10)')
|
|
244
|
+
# There are tests for the objects and types also:
|
|
245
|
+
self.assertTrue(is_planobject(obj))
|
|
246
|
+
self.assertFalse(is_planobject(None))
|
|
247
|
+
self.assertTrue(is_plantype(SimpleObj))
|
|
248
|
+
self.assertFalse(is_plantype(type))
|