h2lib 13.0.703__cp311-cp311-win_amd64.whl → 13.0.705__cp311-cp311-win_amd64.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- h2lib/HAWC2Lib.dll +0 -0
- h2lib/_h2lib.py +42 -15
- h2lib/_version.py +3 -3
- h2lib/dll_wrapper.py +80 -74
- h2lib/h2lib_signatures.py +11 -5
- {h2lib-13.0.703.dist-info → h2lib-13.0.705.dist-info}/METADATA +2 -2
- h2lib-13.0.705.dist-info/RECORD +14 -0
- {h2lib-13.0.703.dist-info → h2lib-13.0.705.dist-info}/WHEEL +1 -1
- h2lib-13.0.705.dist-info/top_level.txt +2 -0
- multiclass_interface/mpi_interface.py +184 -0
- multiclass_interface/multi_object_list.py +57 -0
- multiclass_interface/multiprocess_interface.py +184 -0
- {h2lib → multiclass_interface}/my_test_cls.py +4 -0
- h2lib/mpi_utils.py +0 -146
- h2lib/utils.py +0 -183
- h2lib-13.0.703.dist-info/RECORD +0 -13
- h2lib-13.0.703.dist-info/top_level.txt +0 -1
h2lib/HAWC2Lib.dll
CHANGED
Binary file
|
h2lib/_h2lib.py
CHANGED
@@ -1,8 +1,11 @@
|
|
1
|
-
import
|
2
|
-
import numpy as np
|
1
|
+
from contextlib import contextmanager
|
3
2
|
from h2lib.dll_wrapper import DLLWrapper
|
4
3
|
from h2lib.h2lib_signatures import H2LibSignatures
|
5
|
-
|
4
|
+
import os
|
5
|
+
import sys
|
6
|
+
|
7
|
+
from multiclass_interface.multiprocess_interface import MultiProcessClassInterface, ProcessClass
|
8
|
+
import numpy as np
|
6
9
|
|
7
10
|
|
8
11
|
def H2Lib(suppress_output=False, subprocess=True):
|
@@ -23,6 +26,9 @@ class H2LibThread(H2LibSignatures, DLLWrapper):
|
|
23
26
|
# doubles the speed of single instances and 2N of N instances on linux
|
24
27
|
os.environ['MKL_THREADING_LAYER'] = 'sequential'
|
25
28
|
filename = os.path.abspath(filename)
|
29
|
+
library_bin = os.path.abspath(os.path.join(os.path.dirname(sys.executable), '../Library/bin'))
|
30
|
+
if os.path.isdir(library_bin): # pragma: no cover
|
31
|
+
os.add_dll_directory(library_bin)
|
26
32
|
DLLWrapper.__init__(self, filename, cwd=cwd, cdecl=True)
|
27
33
|
self.suppress_output = suppress_output
|
28
34
|
self._initialized = False
|
@@ -42,6 +48,7 @@ class H2LibThread(H2LibSignatures, DLLWrapper):
|
|
42
48
|
if self._initialized:
|
43
49
|
self.finalize()
|
44
50
|
DLLWrapper.close(self)
|
51
|
+
return "closed h2lib"
|
45
52
|
|
46
53
|
def getState(self):
|
47
54
|
return H2LibSignatures.getState(self, restype=np.int32)[1]
|
@@ -108,11 +115,11 @@ class H2LibThread(H2LibSignatures, DLLWrapper):
|
|
108
115
|
sensor_line += ";"
|
109
116
|
index_start, index_stop = 0, 0
|
110
117
|
index_start, index_stop = H2LibSignatures.add_sensor(self, sensor_line.lower(), index_start, index_stop)[0][1:]
|
111
|
-
return
|
118
|
+
return tuple(range(index_start, index_stop + 1))
|
112
119
|
|
113
120
|
def get_sensor_info(self, id):
|
114
121
|
"return name, unit, description"
|
115
|
-
if isinstance(id,
|
122
|
+
if isinstance(id, tuple):
|
116
123
|
return [self.get_sensor_info(i) for i in id]
|
117
124
|
return [s[:-1].strip() # remove null termination
|
118
125
|
for s in H2LibSignatures.get_sensor_info(self, id, name=" " * 30, unit=" " * 10, desc=" " * 512)[0][1:]]
|
@@ -201,7 +208,7 @@ class H2LibThread(H2LibSignatures, DLLWrapper):
|
|
201
208
|
"""
|
202
209
|
if time is None:
|
203
210
|
time = self.get_time()
|
204
|
-
return H2LibSignatures.set_windfield(self, np.asarray(uvw, dtype=np.
|
211
|
+
return H2LibSignatures.set_windfield(self, np.asarray(uvw, dtype=np.float32),
|
205
212
|
np.float64(box_offset_x), np.float64(time))
|
206
213
|
|
207
214
|
# ===================================================================================================================
|
@@ -230,7 +237,7 @@ class H2LibThread(H2LibSignatures, DLLWrapper):
|
|
230
237
|
return self._aero_sections_data_shape[rotor]
|
231
238
|
|
232
239
|
def get_aerosections_position(self, rotor=0):
|
233
|
-
position = np.zeros(self.aero_sections_data_shape(rotor), dtype=np.float64)
|
240
|
+
position = np.zeros(self.aero_sections_data_shape(rotor), dtype=np.float64, order='F')
|
234
241
|
return H2LibSignatures.get_aerosections_position(self, rotor + 1, position)[0][1]
|
235
242
|
|
236
243
|
def set_aerosections_windspeed(self, uvw, rotor=0):
|
@@ -238,12 +245,12 @@ class H2LibThread(H2LibSignatures, DLLWrapper):
|
|
238
245
|
|
239
246
|
def get_aerosections_forces(self, rotor=0):
|
240
247
|
shape = self.aero_sections_data_shape(rotor)
|
241
|
-
Fxyz = np.zeros(shape, dtype=np.float64)
|
248
|
+
Fxyz = np.zeros(shape, dtype=np.float64, order='F')
|
242
249
|
return H2LibSignatures.get_aerosections_forces(self, rotor + 1, Fxyz)[0][1]
|
243
250
|
|
244
251
|
def get_aerosections_moments(self, rotor=0):
|
245
252
|
shape = self.aero_sections_data_shape(rotor)
|
246
|
-
Mxyz = np.zeros(shape, dtype=np.float64)
|
253
|
+
Mxyz = np.zeros(shape, dtype=np.float64, order='F')
|
247
254
|
return H2LibSignatures.get_aerosections_moments(self, rotor + 1, Mxyz)[0][1]
|
248
255
|
|
249
256
|
def get_bem_grid_dim(self, rotor=0):
|
@@ -254,11 +261,12 @@ class H2LibThread(H2LibSignatures, DLLWrapper):
|
|
254
261
|
"""returns azi, rad"""
|
255
262
|
nazi, nrad = self.get_bem_grid_dim(rotor)
|
256
263
|
return H2LibSignatures.get_bem_grid(self, rotor + 1,
|
257
|
-
np.zeros(nazi, dtype=np.float64
|
264
|
+
np.zeros(nazi, dtype=np.float64, order='F'),
|
265
|
+
np.zeros(nrad, dtype=np.float64, order='F'))[0][1:]
|
258
266
|
|
259
267
|
def get_induction_polargrid(self, rotor=0):
|
260
268
|
nazi, nrad = self.get_bem_grid_dim(rotor)
|
261
|
-
induction = np.zeros((nazi, nrad), dtype=np.float64)
|
269
|
+
induction = np.zeros((nazi, nrad), dtype=np.float64, order='F')
|
262
270
|
return H2LibSignatures.get_induction_polargrid(self, rotor + 1, induction)[0][1]
|
263
271
|
|
264
272
|
def get_induction_axisymmetric(self, rotor=0):
|
@@ -266,6 +274,10 @@ class H2LibThread(H2LibSignatures, DLLWrapper):
|
|
266
274
|
induction = np.zeros(nrad, dtype=np.float64)
|
267
275
|
return H2LibSignatures.get_induction_axisymmetric(self, rotor + 1, induction)[0][1]
|
268
276
|
|
277
|
+
def get_induction_rotoravg(self, rotor=0):
|
278
|
+
induction = np.float64(0)
|
279
|
+
return H2LibSignatures.get_induction_rotoravg(self, rotor + 1, induction)[0][1]
|
280
|
+
|
269
281
|
def get_rotor_orientation(self, rotor=0, deg=False):
|
270
282
|
"""return yaw, tilt, azi(of first blade) in rad(default) or deg"""
|
271
283
|
r = H2LibSignatures.get_rotor_orientation(self, rotor=rotor + 1, yaw=0., tilt=0., azi=0.)[0][1:]
|
@@ -288,6 +300,21 @@ class H2LibThread(H2LibSignatures, DLLWrapper):
|
|
288
300
|
return [vy, vx, -vz]
|
289
301
|
|
290
302
|
|
303
|
+
@contextmanager
|
304
|
+
def set_LD_LIBRARY_PATH():
|
305
|
+
_file__ = np.__file__
|
306
|
+
old = os.environ.get('LD_LIBRARY_PATH', "")
|
307
|
+
|
308
|
+
if '/lib/' in _file__: # pragma: no cover (only on linux)
|
309
|
+
lib_path = _file__[:_file__.index("/lib/") + 5]
|
310
|
+
os.environ['LD_LIBRARY_PATH'] = f'{lib_path}:{old}'
|
311
|
+
|
312
|
+
try:
|
313
|
+
yield
|
314
|
+
finally:
|
315
|
+
os.environ['LD_LIBRARY_PATH'] = old
|
316
|
+
|
317
|
+
|
291
318
|
class H2LibProcess(ProcessClass, H2LibThread):
|
292
319
|
def __init__(self, suppress_output, filename=None, cwd='.'):
|
293
320
|
with set_LD_LIBRARY_PATH():
|
@@ -302,8 +329,8 @@ def MultiH2Lib(N, filename=None, cwd='.', suppress_output=False):
|
|
302
329
|
args = [(suppress_output[i], filename, cwd) for i in range(N)]
|
303
330
|
|
304
331
|
with set_LD_LIBRARY_PATH():
|
305
|
-
from
|
306
|
-
if
|
332
|
+
from multiclass_interface import mpi_interface
|
333
|
+
if mpi_interface.size > 1: # pragma: no cover
|
307
334
|
# try:
|
308
335
|
# with H2LibThread(filename):
|
309
336
|
# pass
|
@@ -312,8 +339,8 @@ def MultiH2Lib(N, filename=None, cwd='.', suppress_output=False):
|
|
312
339
|
# except OSError:
|
313
340
|
# # Set LD_LIBRARY_PATH in mpi workers and run H2LibThread from the workers via ProcessClass
|
314
341
|
# cls = H2LibProcess
|
315
|
-
from
|
342
|
+
from multiclass_interface.mpi_interface import MPIClassInterface
|
316
343
|
cls = H2LibProcess
|
317
344
|
return MPIClassInterface(cls, args)
|
318
345
|
else:
|
319
|
-
return
|
346
|
+
return MultiProcessClassInterface(H2LibThread, args)
|
h2lib/_version.py
CHANGED
@@ -1,4 +1,4 @@
|
|
1
1
|
# This file is autogenerated and should not be modified manually
|
2
|
-
__version__ = '13.0.
|
3
|
-
h2lib_version = '13.0.
|
4
|
-
hawc2_version = '13.0.
|
2
|
+
__version__ = '13.0.705'
|
3
|
+
h2lib_version = '13.0.705'
|
4
|
+
hawc2_version = '13.0.9'
|
h2lib/dll_wrapper.py
CHANGED
@@ -7,8 +7,6 @@ import os
|
|
7
7
|
import ctypes
|
8
8
|
from _ctypes import POINTER
|
9
9
|
from ctypes import c_int, c_double, c_char, c_char_p, c_long, c_longlong
|
10
|
-
import tempfile
|
11
|
-
import shutil
|
12
10
|
from contextlib import contextmanager
|
13
11
|
try:
|
14
12
|
from ctypes import windll
|
@@ -21,6 +19,7 @@ c_int_p = POINTER(ctypes.c_long)
|
|
21
19
|
c_long_p = POINTER(ctypes.c_longlong)
|
22
20
|
|
23
21
|
c_double_p = POINTER(ctypes.c_double)
|
22
|
+
c_float_p = POINTER(ctypes.c_float)
|
24
23
|
|
25
24
|
in_use = []
|
26
25
|
|
@@ -70,6 +69,77 @@ def cwd(f):
|
|
70
69
|
return wrap
|
71
70
|
|
72
71
|
|
72
|
+
def wrap(self, f, *args, **kwargs):
|
73
|
+
c_args = []
|
74
|
+
args = list(args)
|
75
|
+
for i, arg in enumerate(args):
|
76
|
+
if isinstance(arg, (list, tuple)):
|
77
|
+
if all([isinstance(e, int) for e in arg]):
|
78
|
+
# default to int64 which is default on linux but not windows
|
79
|
+
args[i] = np.array(arg, dtype=np.int64)
|
80
|
+
else:
|
81
|
+
args[i] = np.array(arg)
|
82
|
+
if isinstance(args[i], np.ndarray):
|
83
|
+
|
84
|
+
if self.fortran:
|
85
|
+
if not args[i].flags.f_contiguous:
|
86
|
+
sys.stderr.write(f'argument {i} for {f.__name__} not f_contiguous\n')
|
87
|
+
else:
|
88
|
+
if not args[i].flags.c_contiguous:
|
89
|
+
sys.stderr.write(f'argument {i} for {f.__name__} not c_contiguous\n')
|
90
|
+
|
91
|
+
args[i] = np.require(args[i], requirements=['C', 'F'][self.fortran])
|
92
|
+
|
93
|
+
for arg in args:
|
94
|
+
if isinstance(arg, int):
|
95
|
+
c_args.append(c_long_p(c_longlong(arg)))
|
96
|
+
elif isinstance(arg, float):
|
97
|
+
c_args.append(c_double_p(c_double(arg)))
|
98
|
+
elif isinstance(arg, str):
|
99
|
+
c_args.append(c_char_p(arg.encode('cp1252')))
|
100
|
+
# c_args.append(c_int_p(c_int(len(arg))))
|
101
|
+
|
102
|
+
elif isinstance(arg, np.ndarray):
|
103
|
+
if arg.dtype in [np.int32]:
|
104
|
+
c_args.append(arg.ctypes.data_as(c_int_p))
|
105
|
+
elif arg.dtype in [np.int64]:
|
106
|
+
c_args.append(arg.ctypes.data_as(c_long_p))
|
107
|
+
elif arg.dtype == np.float64:
|
108
|
+
c_args.append(arg.ctypes.data_as(c_double_p))
|
109
|
+
elif arg.dtype == np.float32:
|
110
|
+
c_args.append(arg.ctypes.data_as(c_float_p))
|
111
|
+
else:
|
112
|
+
raise NotImplementedError(arg.dtype)
|
113
|
+
|
114
|
+
else:
|
115
|
+
# raise NotImplementedError(arg.__class__.__name__)
|
116
|
+
c_args.append(arg)
|
117
|
+
if 'restype' in kwargs:
|
118
|
+
restype = kwargs['restype']
|
119
|
+
if hasattr(restype, 'dtype'):
|
120
|
+
restype = np.ctypeslib.as_ctypes_type(restype)
|
121
|
+
f.restype = restype
|
122
|
+
with chdir(self.cwd):
|
123
|
+
if self.suppress_output:
|
124
|
+
with SuppressStream(sys.stdout), SuppressStream(sys.stderr):
|
125
|
+
res = f(*c_args)
|
126
|
+
else:
|
127
|
+
res = f(*c_args)
|
128
|
+
ret_args = []
|
129
|
+
for arg in args:
|
130
|
+
c_arg = c_args.pop(0)
|
131
|
+
if isinstance(arg, (int, float)):
|
132
|
+
ret_args.append(c_arg.contents.value)
|
133
|
+
elif isinstance(arg, (str)):
|
134
|
+
ret_args.append(c_arg.value.decode('cp1252'))
|
135
|
+
# c_args.pop(0)
|
136
|
+
elif isinstance(arg, np.ndarray):
|
137
|
+
ret_args.append(arg)
|
138
|
+
else:
|
139
|
+
raise NotImplementedError(arg.__class__.__name__)
|
140
|
+
return ret_args, res
|
141
|
+
|
142
|
+
|
73
143
|
class DLLWrapper(object):
|
74
144
|
def __init__(self, filename, cwd='.', cdecl=True, fortran=True):
|
75
145
|
self.filename = str(filename)
|
@@ -124,9 +194,10 @@ class DLLWrapper(object):
|
|
124
194
|
def open(self):
|
125
195
|
assert os.path.isfile(self.filename), os.path.abspath(self.filename)
|
126
196
|
if self.cdecl:
|
127
|
-
|
197
|
+
try:
|
198
|
+
# python < (3, 8) and > 3.10?:
|
128
199
|
self.lib = ct.CDLL(self.filename)
|
129
|
-
|
200
|
+
except BaseException:
|
130
201
|
self.lib = ct.CDLL(self.filename, winmode=ctypes.DEFAULT_MODE)
|
131
202
|
else:
|
132
203
|
self.lib = windll.LoadLibrary(self.filename)
|
@@ -148,82 +219,17 @@ class DLLWrapper(object):
|
|
148
219
|
# self.close()
|
149
220
|
# return False
|
150
221
|
|
151
|
-
def
|
152
|
-
|
153
|
-
|
154
|
-
|
155
|
-
if name == 'lib':
|
156
|
-
raise Exception("DLL not loaded. Run using: 'with dll: ...'")
|
157
|
-
return self.get_lib_function(name)
|
222
|
+
def __getattr__(self, name):
|
223
|
+
if name == 'lib':
|
224
|
+
raise Exception("DLL not loaded. Run using: 'with dll: ...'")
|
225
|
+
return self.get_lib_function(name)
|
158
226
|
|
159
227
|
def get_lib_function(self, name):
|
160
228
|
try:
|
161
229
|
f = getattr(self.lib, name)
|
162
230
|
except AttributeError as e:
|
163
231
|
raise AttributeError("Attribute '%s' not found in dll ('%s')" % (name, self.filename))
|
164
|
-
|
165
|
-
def wrap(*args, **kwargs):
|
166
|
-
c_args = []
|
167
|
-
args = list(args)
|
168
|
-
for i, arg in enumerate(args):
|
169
|
-
if isinstance(arg, (list, tuple)):
|
170
|
-
if all([isinstance(e, int) for e in arg]):
|
171
|
-
# default to int64 which is default on linux but not windows
|
172
|
-
args[i] = np.array(arg, dtype=np.int64)
|
173
|
-
else:
|
174
|
-
args[i] = np.array(arg)
|
175
|
-
if isinstance(args[i], np.ndarray):
|
176
|
-
if self.fortran:
|
177
|
-
args[i] = np.asfortranarray(args[i])
|
178
|
-
else:
|
179
|
-
args[i] = np.ascontiguousarray(args[i])
|
180
|
-
for arg in args:
|
181
|
-
if isinstance(arg, int):
|
182
|
-
c_args.append(c_long_p(c_longlong(arg)))
|
183
|
-
elif isinstance(arg, float):
|
184
|
-
c_args.append(c_double_p(c_double(arg)))
|
185
|
-
elif isinstance(arg, str):
|
186
|
-
c_args.append(c_char_p(arg.encode('cp1252')))
|
187
|
-
# c_args.append(c_int_p(c_int(len(arg))))
|
188
|
-
|
189
|
-
elif isinstance(arg, np.ndarray):
|
190
|
-
if arg.dtype in [np.int32]:
|
191
|
-
c_args.append(arg.ctypes.data_as(c_int_p))
|
192
|
-
elif arg.dtype in [np.int64]:
|
193
|
-
c_args.append(arg.ctypes.data_as(c_long_p))
|
194
|
-
elif arg.dtype == np.float64:
|
195
|
-
c_args.append(arg.ctypes.data_as(c_double_p))
|
196
|
-
else:
|
197
|
-
raise NotImplementedError(arg.dtype)
|
198
|
-
|
199
|
-
else:
|
200
|
-
# raise NotImplementedError(arg.__class__.__name__)
|
201
|
-
c_args.append(arg)
|
202
|
-
if 'restype' in kwargs:
|
203
|
-
restype = kwargs['restype']
|
204
|
-
if hasattr(restype, 'dtype'):
|
205
|
-
restype = np.ctypeslib.as_ctypes_type(restype)
|
206
|
-
f.restype = restype
|
207
|
-
with chdir(self.cwd):
|
208
|
-
if self.suppress_output:
|
209
|
-
with SuppressStream(sys.stdout), SuppressStream(sys.stderr):
|
210
|
-
res = f(*c_args)
|
211
|
-
else:
|
212
|
-
res = f(*c_args)
|
213
|
-
ret_args = []
|
214
|
-
for arg in args:
|
215
|
-
c_arg = c_args.pop(0)
|
216
|
-
if isinstance(arg, (int, float)):
|
217
|
-
ret_args.append(c_arg.contents.value)
|
218
|
-
elif isinstance(arg, (str)):
|
219
|
-
ret_args.append(c_arg.value.decode('cp1252'))
|
220
|
-
# c_args.pop(0)
|
221
|
-
elif isinstance(arg, np.ndarray):
|
222
|
-
ret_args.append(arg)
|
223
|
-
else:
|
224
|
-
raise NotImplementedError(arg.__class__.__name__)
|
225
|
-
return ret_args, res
|
226
|
-
return wrap
|
232
|
+
return lambda *args, **kwargs: wrap(self, f, *args, **kwargs)
|
227
233
|
|
228
234
|
def version(self, function_name='get_version'):
|
229
235
|
try:
|
h2lib/h2lib_signatures.py
CHANGED
@@ -121,6 +121,14 @@ END SUBROUTINE'''
|
|
121
121
|
end subroutine'''
|
122
122
|
return self.get_lib_function('get_induction_polargrid')(rotor, induction)
|
123
123
|
|
124
|
+
def get_induction_rotoravg(self, rotor, induction):
|
125
|
+
'''subroutine get_induction_rotoravg(rotor, induction) bind(C, name="get_induction_rotoravg")
|
126
|
+
!DEC$ ATTRIBUTES DLLEXPORT :: get_induction_rotoravg
|
127
|
+
integer*8, intent(in) :: rotor
|
128
|
+
real(c_double), intent(out) :: induction
|
129
|
+
end subroutine'''
|
130
|
+
return self.get_lib_function('get_induction_rotoravg')(rotor, induction)
|
131
|
+
|
124
132
|
def get_nSections(self, rotor, blade, restype):
|
125
133
|
'''function get_nSections(rotor, blade) bind(C, name="get_nSections")
|
126
134
|
!DEC$ ATTRIBUTES DLLEXPORT :: get_nSections
|
@@ -187,10 +195,8 @@ END SUBROUTINE'''
|
|
187
195
|
return self.get_lib_function('get_sensor_values')(ids, values, n)
|
188
196
|
|
189
197
|
def get_time(self, time):
|
190
|
-
'''subroutine
|
191
|
-
|
192
|
-
double precision :: time ! time [s]
|
193
|
-
end subroutine'''
|
198
|
+
'''subroutine
|
199
|
+
subroutine'''
|
194
200
|
return self.get_lib_function('get_time')(time)
|
195
201
|
|
196
202
|
def get_version(self, s):
|
@@ -273,7 +279,7 @@ END SUBROUTINE'''
|
|
273
279
|
|
274
280
|
def set_windfield(self, uvw, box_offset_x, time):
|
275
281
|
'''subroutine set_windfield(uvw, box_offset_x, time) bind(C, name="set_windfield")
|
276
|
-
real*
|
282
|
+
real*4, intent(in) :: uvw(3, gwsd%TMOD%buffer_points_x,gwsd%TMOD%buffer_points_y,gwsd%TMOD%buffer_points_z)
|
277
283
|
real*8, intent(in):: box_offset_x, time
|
278
284
|
end subroutine'''
|
279
285
|
return self.get_lib_function('set_windfield')(uvw, box_offset_x, time)
|
@@ -1,7 +1,7 @@
|
|
1
1
|
Metadata-Version: 2.1
|
2
2
|
Name: h2lib
|
3
|
-
Version: 13.0.
|
4
|
-
Summary: Python interface to HAWC2 (13.0.
|
3
|
+
Version: 13.0.705
|
4
|
+
Summary: Python interface to HAWC2 (13.0.9)
|
5
5
|
Download-URL:
|
6
6
|
Author: Mads M. Pedersen, S.G.Horcas and N.G.Ramos
|
7
7
|
Author-email: mmpe@dtu.dk
|
@@ -0,0 +1,14 @@
|
|
1
|
+
h2lib/HAWC2Lib.dll,sha256=GhnrcvnvBrC_KxHN2Asm1U7Ak6SW1axLhiBGPocwAvw,29301760
|
2
|
+
h2lib/__init__.py,sha256=f3fO4I6IEFRM9LaV2O3w9Pioj3GPI8qRl7P5Tg5ONtE,528
|
3
|
+
h2lib/_h2lib.py,sha256=PvXyuoVW6IDbjeYWYtCHRYcpLMlZh7aCiJkfLBl88gg,14938
|
4
|
+
h2lib/_version.py,sha256=xyyOn5ozEvCx53n7O9Ht13YA4b22VWeehuYZqCSUdEk,146
|
5
|
+
h2lib/dll_wrapper.py,sha256=vO7IGbnPTpzrbjsvm-jRnYabXM97gCukJDTPJHDLEpA,11824
|
6
|
+
h2lib/h2lib_signatures.py,sha256=uIcqvgVbm_L5C4jpHc9mBqz_6G5sLIk_6n2i-j1oA04,14500
|
7
|
+
multiclass_interface/mpi_interface.py,sha256=eGftiuyQ3QkcenpEmbZjy_3VomWoXADnQsGUCFoFBvs,6274
|
8
|
+
multiclass_interface/multi_object_list.py,sha256=5bEdwvtzQkPyBrS-W64i3EHJmuTfNGmRhGm5ToK7SXI,2072
|
9
|
+
multiclass_interface/multiprocess_interface.py,sha256=L5G7-EL5rXQtze5-4oSTBFs0AjswbD2diNtt0ynfWOM,7271
|
10
|
+
multiclass_interface/my_test_cls.py,sha256=7ZDsFkxrLfOY6q00U5Y-daxfuhATK-K5H04RP-VmQdE,850
|
11
|
+
h2lib-13.0.705.dist-info/METADATA,sha256=54FEPqcr2y_3vR70shyoAVXYpWfUwFdJcDNWW-EFR3M,623
|
12
|
+
h2lib-13.0.705.dist-info/WHEEL,sha256=ircjsfhzblqgSzO8ow7-0pXK-RVqDqNRGQ8F650AUNM,102
|
13
|
+
h2lib-13.0.705.dist-info/top_level.txt,sha256=CafRr3oTgH80oaQrp2SGKlPcX7cag5ag4EGKAZAy1ow,27
|
14
|
+
h2lib-13.0.705.dist-info/RECORD,,
|
@@ -0,0 +1,184 @@
|
|
1
|
+
import numpy as np
|
2
|
+
import sys
|
3
|
+
import os
|
4
|
+
import subprocess
|
5
|
+
import traceback
|
6
|
+
import inspect
|
7
|
+
from enum import Enum
|
8
|
+
|
9
|
+
if 'SLURM_NTASKS' in os.environ:
|
10
|
+
mpi = int(os.environ['SLURM_NTASKS']) > 1
|
11
|
+
elif any([k in os.environ for k in ['MPI_LOCALNRANKS', 'OMPI_COMM_WORLD_SIZE']]):
|
12
|
+
mpi = True
|
13
|
+
else:
|
14
|
+
mpi = subprocess.run("python -c 'from mpi4py import MPI'", shell=True,
|
15
|
+
check=False, stderr=subprocess.PIPE).returncode == 0
|
16
|
+
|
17
|
+
if mpi:
|
18
|
+
from mpi4py import MPI
|
19
|
+
comm = MPI.COMM_WORLD
|
20
|
+
size = MPI.COMM_WORLD.Get_size()
|
21
|
+
rank = MPI.COMM_WORLD.Get_rank()
|
22
|
+
name = MPI.Get_processor_name()
|
23
|
+
else:
|
24
|
+
size = 1
|
25
|
+
|
26
|
+
MPIClassInterfaceAttributes = {'close', 'use_rank', 'cls', 'work_loop', 'object', '__class__','get_input','do_task', 'run_task', 'closed'}
|
27
|
+
|
28
|
+
|
29
|
+
LOOP_UNTIL_CLOSE=True
|
30
|
+
TERMINATE_ON_CLOSE=True
|
31
|
+
|
32
|
+
|
33
|
+
class MPIClassInterface():
|
34
|
+
def __init__(self, cls, args_lst):
|
35
|
+
if len(args_lst) > size:
|
36
|
+
if rank == 0:
|
37
|
+
raise Exception(f"Not enough mpi slots. Slots: {size}, Requested: {len(args_lst)}")
|
38
|
+
return
|
39
|
+
self.cls = cls
|
40
|
+
if rank < len(args_lst):
|
41
|
+
self.object = cls(*args_lst[rank])
|
42
|
+
else:
|
43
|
+
class Dummy():
|
44
|
+
def close(self):
|
45
|
+
pass
|
46
|
+
self.object = Dummy()
|
47
|
+
self.closed=False
|
48
|
+
|
49
|
+
if rank == 0:
|
50
|
+
self.use_rank = np.array([True] * size)
|
51
|
+
self.use_rank[len(args_lst):] = False
|
52
|
+
elif LOOP_UNTIL_CLOSE:
|
53
|
+
self.work_loop()
|
54
|
+
|
55
|
+
def __enter__(self):
|
56
|
+
return self
|
57
|
+
|
58
|
+
def __exit__(self, exc_type, exc_val, exc_tb):
|
59
|
+
self.close()
|
60
|
+
|
61
|
+
def work_loop(self):
|
62
|
+
while True:
|
63
|
+
method, args, kwargs = comm.scatter(None)
|
64
|
+
comm.gather(self.do_task(method, args, kwargs))
|
65
|
+
if LOOP_UNTIL_CLOSE:
|
66
|
+
if method == 'close':
|
67
|
+
if TERMINATE_ON_CLOSE:
|
68
|
+
#comm.gather(f'Exit, rank {rank}')
|
69
|
+
print ("sys.exit", rank, flush=True)
|
70
|
+
sys.exit(0)
|
71
|
+
else:
|
72
|
+
raise ChildProcessError('MPI worker done')
|
73
|
+
else:
|
74
|
+
break
|
75
|
+
|
76
|
+
|
77
|
+
def do_task(self, method, args, kwargs):
|
78
|
+
try:
|
79
|
+
if method == 'skip':
|
80
|
+
res = None
|
81
|
+
elif method=='setattr':
|
82
|
+
name, value = args
|
83
|
+
if hasattr(value, '__call__'):
|
84
|
+
def wrap(*args, func=value, **kwargs):
|
85
|
+
if inspect.getfullargspec(func).args[:1] == ['self']:
|
86
|
+
args = (self.object,) + args
|
87
|
+
return func(*args, **kwargs)
|
88
|
+
value = wrap
|
89
|
+
res = setattr(self.object, name, value)
|
90
|
+
else:
|
91
|
+
res = getattr(self.object, method)
|
92
|
+
if hasattr(res, '__call__'):
|
93
|
+
res = res(*args, **kwargs)
|
94
|
+
except BaseException as e:
|
95
|
+
res = (e, traceback.format_exc())
|
96
|
+
return res
|
97
|
+
|
98
|
+
def run_task(self, input_lst):
|
99
|
+
comm.scatter(input_lst, root=0)
|
100
|
+
if rank == 0:
|
101
|
+
method, args, kwargs = input_lst[0]
|
102
|
+
res = self.do_task(method, args, kwargs)
|
103
|
+
else:
|
104
|
+
res = None
|
105
|
+
use_rank = self.use_rank
|
106
|
+
|
107
|
+
res = [res for i, res in enumerate(comm.gather(res, root=0)) if use_rank[i]]
|
108
|
+
if rank==0:
|
109
|
+
for r in res:
|
110
|
+
if isinstance(r, tuple) and len(r) > 1 and isinstance(r[0], BaseException):
|
111
|
+
raise r[0].__class__(r[1])
|
112
|
+
return res
|
113
|
+
|
114
|
+
|
115
|
+
def get_input(self, name, i, args, kwargs):
|
116
|
+
use_rank = self.use_rank
|
117
|
+
N = np.sum(use_rank)
|
118
|
+
if use_rank[i]:
|
119
|
+
def get_arg(arg):
|
120
|
+
if isinstance(arg, list) and len(arg) == N:
|
121
|
+
return arg[np.where(use_rank)[0][i]]
|
122
|
+
else:
|
123
|
+
return arg
|
124
|
+
return (name, [get_arg(arg) for arg in args], {k: get_arg(v) for k, v in kwargs.items()})
|
125
|
+
else:
|
126
|
+
return ('skip', [], {})
|
127
|
+
|
128
|
+
def __getattribute__(self, name):
|
129
|
+
if name in MPIClassInterfaceAttributes:
|
130
|
+
return object.__getattribute__(self, name)
|
131
|
+
elif rank > 0:
|
132
|
+
self.work_loop()
|
133
|
+
return lambda *args, **kwargs:1
|
134
|
+
|
135
|
+
def wrap(*args, **kwargs):
|
136
|
+
inp = [self.get_input(name, i, args, kwargs) for i in range(size)]
|
137
|
+
return self.run_task(inp)
|
138
|
+
|
139
|
+
if hasattr(getattr(self.object, name), '__call__'):
|
140
|
+
return wrap
|
141
|
+
else:
|
142
|
+
return wrap()
|
143
|
+
|
144
|
+
def __setattr__(self, name, value):
|
145
|
+
if rank > 0 or name in MPIClassInterfaceAttributes:
|
146
|
+
return object.__setattr__(self, name, value)
|
147
|
+
else:
|
148
|
+
inp = [self.get_input('setattr', i, (name, value), {}) for i in range(size)]
|
149
|
+
return self.run_task(inp)
|
150
|
+
|
151
|
+
|
152
|
+
def __getitem__(self, slice):
|
153
|
+
use_rank = np.full_like(self.use_rank, False)
|
154
|
+
use_rank[slice] = True
|
155
|
+
if np.all(self.use_rank == use_rank):
|
156
|
+
return self
|
157
|
+
return SubsetMPIClassInterface(self.cls, self.object, use_rank)
|
158
|
+
|
159
|
+
def close(self):
|
160
|
+
if not self.closed:
|
161
|
+
if rank==0:
|
162
|
+
res = self.run_task([('close', [], {}) for _ in range(size)])
|
163
|
+
else:
|
164
|
+
self.work_loop()
|
165
|
+
res = None
|
166
|
+
self.closed=True
|
167
|
+
|
168
|
+
|
169
|
+
|
170
|
+
|
171
|
+
class SubsetMPIClassInterface(MPIClassInterface):
|
172
|
+
def __init__(self, cls, object, use_rank):
|
173
|
+
self.use_rank = use_rank
|
174
|
+
self.cls = cls
|
175
|
+
self.object = object
|
176
|
+
|
177
|
+
def __getitem__(self, slice):
|
178
|
+
l = np.arange(np.sum(self.use_rank))
|
179
|
+
if np.all(l == l[slice]):
|
180
|
+
return self
|
181
|
+
raise Exception('Cannot make subset of SubsetMPIClassInterface')
|
182
|
+
|
183
|
+
def close(self):
|
184
|
+
raise Exception('Cannot close SubsetMPIClassInterface. Please close all instances at once')
|
@@ -0,0 +1,57 @@
|
|
1
|
+
import numpy as np
|
2
|
+
|
3
|
+
|
4
|
+
class MultiObjectList():
|
5
|
+
def __init__(self, obj_lst, subset_cls=None):
|
6
|
+
self.obj_lst = np.atleast_1d(obj_lst)
|
7
|
+
self.subset_cls = subset_cls
|
8
|
+
|
9
|
+
def __str__(self):
|
10
|
+
return f'{self.__class__.__name__}({self.obj_lst})'
|
11
|
+
|
12
|
+
def get_obj_args_lst(self, args, kwargs):
|
13
|
+
N = len(self.obj_lst)
|
14
|
+
|
15
|
+
def get_obj_args(i):
|
16
|
+
def get_arg(arg):
|
17
|
+
if isinstance(arg, list) and len(arg) == N:
|
18
|
+
return arg[i]
|
19
|
+
else:
|
20
|
+
return arg
|
21
|
+
obj_args = [get_arg(arg) for arg in args]
|
22
|
+
obj_kwargs = {k: get_arg(v) for k, v in kwargs.items()}
|
23
|
+
return obj_args, obj_kwargs
|
24
|
+
return [get_obj_args(i) for i in range(N)]
|
25
|
+
|
26
|
+
def iscallable(self, name):
|
27
|
+
return hasattr(getattr(self.obj_lst[0], name), '__call__')
|
28
|
+
|
29
|
+
def __getitem__(self, s):
|
30
|
+
obj_lst = np.atleast_1d(self.obj_lst[s])
|
31
|
+
if len(obj_lst) == len(self.obj_lst) and np.all(obj_lst == self.obj_lst):
|
32
|
+
return self
|
33
|
+
subset_cls = self.subset_cls or MultiObjectList
|
34
|
+
return subset_cls(self.obj_lst[s])
|
35
|
+
|
36
|
+
def __getattr__(self, name):
|
37
|
+
att_lst = [getattr(obj, name) for obj in self.obj_lst]
|
38
|
+
if self.iscallable(name):
|
39
|
+
def wrap(*args, **kwargs):
|
40
|
+
return [att(*o_args, **o_kwargs)
|
41
|
+
for att, (o_args, o_kwargs) in zip(att_lst, self.get_obj_args_lst(args, kwargs))]
|
42
|
+
return wrap
|
43
|
+
else:
|
44
|
+
return att_lst
|
45
|
+
|
46
|
+
def __setattr__(self, name, value):
|
47
|
+
if name in {'obj_lst', 'subset_cls'}:
|
48
|
+
return object.__setattr__(self, name, value)
|
49
|
+
obj_lst = self.obj_lst
|
50
|
+
for obj, (o_args, _) in zip(obj_lst, self.get_obj_args_lst((value,), {})):
|
51
|
+
setattr(obj, name, *o_args)
|
52
|
+
|
53
|
+
|
54
|
+
class MultiClassInterface(MultiObjectList):
|
55
|
+
def __init__(self, cls, args_lst):
|
56
|
+
self.cls = cls
|
57
|
+
MultiObjectList.__init__(self, [cls(*args) for args in args_lst])
|
@@ -0,0 +1,184 @@
|
|
1
|
+
import multiprocessing
|
2
|
+
import atexit
|
3
|
+
import numpy as np
|
4
|
+
from threading import Thread
|
5
|
+
from functools import wraps
|
6
|
+
import traceback
|
7
|
+
from _queue import Empty
|
8
|
+
import os
|
9
|
+
from contextlib import contextmanager
|
10
|
+
import sys
|
11
|
+
import inspect
|
12
|
+
from multiclass_interface.multi_object_list import MultiObjectList
|
13
|
+
|
14
|
+
|
15
|
+
def run(cls, inputQueue, outputQueue, cls_args, **kwargs):
|
16
|
+
o = cls(*cls_args, **kwargs)
|
17
|
+
while True:
|
18
|
+
method, args, kwargs = inputQueue.get()
|
19
|
+
try:
|
20
|
+
if method == 'getattr':
|
21
|
+
name = args[0]
|
22
|
+
outputQueue.put(getattr(o, name))
|
23
|
+
elif method == 'setattr':
|
24
|
+
name, value = args
|
25
|
+
if hasattr(value, '__call__'): # pragma: no cover # cov not registered?
|
26
|
+
|
27
|
+
def wrap(*args, func=value, **kwargs):
|
28
|
+
if inspect.getfullargspec(func).args[:1] == ['self']:
|
29
|
+
args = (o,) + args
|
30
|
+
return func(*args, **kwargs)
|
31
|
+
outputQueue.put(setattr(o, name, wrap))
|
32
|
+
else:
|
33
|
+
outputQueue.put(setattr(o, name, value))
|
34
|
+
elif method == 'iscallable':
|
35
|
+
name = args[0]
|
36
|
+
outputQueue.put(hasattr(getattr(o, name), '__call__'))
|
37
|
+
else:
|
38
|
+
att = getattr(o, method)
|
39
|
+
outputQueue.put(att(*args, **kwargs))
|
40
|
+
if method == 'close':
|
41
|
+
outputQueue.put('Exit process')
|
42
|
+
return
|
43
|
+
except BaseException as e:
|
44
|
+
outputQueue.put((e, traceback.format_exc()))
|
45
|
+
|
46
|
+
|
47
|
+
class ProcessClass():
|
48
|
+
cls = None
|
49
|
+
|
50
|
+
def __init__(self, cls, cls_attrs={}):
|
51
|
+
self.cls_attrs = cls_attrs
|
52
|
+
self.cls = cls
|
53
|
+
self.ctx = multiprocessing.get_context('spawn')
|
54
|
+
self.inputQueue = self.ctx.Queue()
|
55
|
+
self.outputQueue = self.ctx.Queue()
|
56
|
+
atexit.register(self.close)
|
57
|
+
self.closed = False
|
58
|
+
|
59
|
+
def __call__(self, *args, **kwargs):
|
60
|
+
kwargs.update({'cls': self.cls, 'inputQueue': self.inputQueue, 'outputQueue': self.outputQueue,
|
61
|
+
'cls_args': args})
|
62
|
+
s = 'vs_debug.py'
|
63
|
+
if s in "".join(traceback.format_stack()): # pragma: no cover
|
64
|
+
self.process = Thread(target=run, kwargs=kwargs) # use this to debug from Visual studio
|
65
|
+
else:
|
66
|
+
self.process = self.ctx.Process(target=run, kwargs=kwargs, daemon=True)
|
67
|
+
|
68
|
+
self.process.start()
|
69
|
+
return self
|
70
|
+
|
71
|
+
def __enter__(self):
|
72
|
+
return self
|
73
|
+
|
74
|
+
def __exit__(self, exc_type, exc_val, exc_tb):
|
75
|
+
self.close()
|
76
|
+
|
77
|
+
def __getattribute__(self, name):
|
78
|
+
try:
|
79
|
+
if name != 'cls_attrs' and name in self.cls_attrs:
|
80
|
+
raise AttributeError()
|
81
|
+
return object.__getattribute__(self, name)
|
82
|
+
except AttributeError:
|
83
|
+
if self.is_callable(name):
|
84
|
+
@wraps(getattr(self.cls, name, None))
|
85
|
+
def wrap(*args, wait_for_result=True, **kwargs):
|
86
|
+
self.inputQueue.put((name, args, kwargs))
|
87
|
+
if wait_for_result:
|
88
|
+
return self.get_result(raise_exception=True,
|
89
|
+
cmd=lambda: f'executing {name}({", ".join(list(map(str,args))+["%s=%s"%(k,v) for k,v in kwargs.items()])})')
|
90
|
+
return wrap
|
91
|
+
else:
|
92
|
+
self.inputQueue.put(('getattr', (name,), {}))
|
93
|
+
return self.get_result(raise_exception=True, cmd=lambda: f"getting attribute '{name}'")
|
94
|
+
|
95
|
+
def is_callable(self, name):
|
96
|
+
self.inputQueue.put(('iscallable', (name,), {}))
|
97
|
+
return self.get_result(raise_exception=True, cmd=lambda: f"checking if '{name}' is callable")
|
98
|
+
|
99
|
+
def __setattr__(self, name, value):
|
100
|
+
if name in {'cls', 'ctx', 'inputQueue', 'outputQueue', 'closed', 'process', 'cls_attrs'}:
|
101
|
+
return object.__setattr__(self, name, value)
|
102
|
+
else:
|
103
|
+
self.inputQueue.put(('setattr', (name, value), {}))
|
104
|
+
return self.get_result(raise_exception=True, cmd=lambda: f"setting attribute '{name}'")
|
105
|
+
|
106
|
+
def get_result(self, raise_exception, cmd):
|
107
|
+
while True:
|
108
|
+
if self.process.is_alive() or self.closed:
|
109
|
+
try:
|
110
|
+
res = self.outputQueue.get(timeout=2)
|
111
|
+
if isinstance(res, tuple) and len(res) > 1 and isinstance(res[0], BaseException):
|
112
|
+
res = res[0].__class__(res[1])
|
113
|
+
if raise_exception:
|
114
|
+
raise res
|
115
|
+
return res
|
116
|
+
except Empty:
|
117
|
+
pass # time out. Check process is alive and try again
|
118
|
+
else:
|
119
|
+
if hasattr(cmd, '__call__'):
|
120
|
+
cmd = cmd()
|
121
|
+
e = Exception(f'{self.cls.__name__} process died before or while {cmd}')
|
122
|
+
if raise_exception:
|
123
|
+
raise e
|
124
|
+
return e
|
125
|
+
|
126
|
+
def close(self, wait_for_result=False):
|
127
|
+
if not self.closed:
|
128
|
+
self.inputQueue.put(('close', [], {}))
|
129
|
+
r = self.get_result(False, 'close')
|
130
|
+
self.process.join()
|
131
|
+
self.inputQueue.close()
|
132
|
+
self.outputQueue.close()
|
133
|
+
self.closed = True
|
134
|
+
return r
|
135
|
+
|
136
|
+
|
137
|
+
class MultiProcessClassInterface(MultiObjectList):
|
138
|
+
|
139
|
+
def __init__(self, cls, args_lst, cls_attrs={}):
|
140
|
+
MultiObjectList.__init__(self, [ProcessClass(cls, cls_attrs)(*args) for args in args_lst], SubsetProcessWrapper)
|
141
|
+
|
142
|
+
def __getattr__(self, name):
|
143
|
+
obj_lst = self.obj_lst
|
144
|
+
if obj_lst[0].is_callable(name):
|
145
|
+
def wrap(*args, **kwargs):
|
146
|
+
for obj, (o_args, o_kwargs) in zip(obj_lst, self.get_obj_args_lst(args, kwargs)):
|
147
|
+
getattr(obj, name)(*o_args, wait_for_result=False, **o_kwargs)
|
148
|
+
res = [o.get_result(raise_exception=False, cmd=lambda: f"executing {name}(...)")
|
149
|
+
for o in obj_lst]
|
150
|
+
for r in res:
|
151
|
+
if isinstance(r, Exception):
|
152
|
+
raise r
|
153
|
+
return res
|
154
|
+
return wrap
|
155
|
+
else:
|
156
|
+
return [getattr(o, name) for o in obj_lst]
|
157
|
+
|
158
|
+
def __enter__(self):
|
159
|
+
return self
|
160
|
+
|
161
|
+
def __exit__(self, exc_type, exc_val, exc_tb):
|
162
|
+
self.close()
|
163
|
+
|
164
|
+
def close(self, wait_for_result=False):
|
165
|
+
for obj in self.obj_lst:
|
166
|
+
obj.inputQueue.put(('close', [], {}))
|
167
|
+
obj.process.join()
|
168
|
+
obj.closed = True
|
169
|
+
|
170
|
+
|
171
|
+
class SubsetProcessWrapper(MultiProcessClassInterface):
|
172
|
+
def __init__(self, obj_lst):
|
173
|
+
MultiObjectList.__init__(self, obj_lst)
|
174
|
+
|
175
|
+
def __getitem__(self, slice):
|
176
|
+
if np.all(np.atleast_1d(self.obj_lst[slice]) == self.obj_lst):
|
177
|
+
return self
|
178
|
+
raise Exception('Cannot make subset of SubsetProcessWrapper')
|
179
|
+
|
180
|
+
def __getattribute__(self, name):
|
181
|
+
if name == 'close':
|
182
|
+
raise Exception("Cannot close SubsetProcessWrapper. Please close all instances at once")
|
183
|
+
|
184
|
+
return MultiProcessClassInterface.__getattribute__(self, name)
|
@@ -5,6 +5,7 @@ import os
|
|
5
5
|
class MyTest():
|
6
6
|
def __init__(self, id):
|
7
7
|
self.id = id
|
8
|
+
self.name = self.__class__.__name__
|
8
9
|
|
9
10
|
def get_id(self,):
|
10
11
|
return self.id
|
@@ -27,3 +28,6 @@ class MyTest():
|
|
27
28
|
|
28
29
|
def close(self):
|
29
30
|
return f"closing {self.get_id()}"
|
31
|
+
|
32
|
+
def raise_exception(self):
|
33
|
+
1 / 0 # raise ZeroDivisionError
|
h2lib/mpi_utils.py
DELETED
@@ -1,146 +0,0 @@
|
|
1
|
-
import numpy as np
|
2
|
-
import sys
|
3
|
-
import os
|
4
|
-
import subprocess
|
5
|
-
|
6
|
-
if 'SLURM_NTASKS' in os.environ:
|
7
|
-
mpi = int(os.environ['SLURM_NTASKS']) > 1
|
8
|
-
elif any([k in os.environ for k in ['MPI_LOCALNRANKS', 'OMPI_COMM_WORLD_SIZE']]):
|
9
|
-
mpi = True
|
10
|
-
else:
|
11
|
-
mpi = subprocess.run("python -c 'from mpi4py import MPI'", shell=True,
|
12
|
-
check=False, stderr=subprocess.PIPE).returncode == 0
|
13
|
-
|
14
|
-
if mpi:
|
15
|
-
from mpi4py import MPI
|
16
|
-
comm = MPI.COMM_WORLD
|
17
|
-
size = MPI.COMM_WORLD.Get_size()
|
18
|
-
rank = MPI.COMM_WORLD.Get_rank()
|
19
|
-
name = MPI.Get_processor_name()
|
20
|
-
else:
|
21
|
-
size = 1
|
22
|
-
|
23
|
-
MPIClassInterfaceAttributes = {'close', 'use_rank', 'cls', 'do_work', 'object', '__class__'}
|
24
|
-
exit_mpi_on_close = True
|
25
|
-
|
26
|
-
|
27
|
-
class MPIClassInterface():
|
28
|
-
def __init__(self, cls, args_lst):
|
29
|
-
if len(args_lst) > size:
|
30
|
-
if rank == 0:
|
31
|
-
raise Exception(f"Not enough mpi slots. Slots: {size}, Requested: {len(args_lst)}")
|
32
|
-
return
|
33
|
-
self.cls = cls
|
34
|
-
if rank < len(args_lst):
|
35
|
-
self.object = cls(*args_lst[rank])
|
36
|
-
else:
|
37
|
-
class Dummy():
|
38
|
-
def close(self):
|
39
|
-
pass
|
40
|
-
self.object = Dummy()
|
41
|
-
|
42
|
-
if rank == 0:
|
43
|
-
self.use_rank = np.array([True] * size)
|
44
|
-
self.use_rank[len(args_lst):] = False
|
45
|
-
else:
|
46
|
-
self.do_work()
|
47
|
-
|
48
|
-
def __enter__(self):
|
49
|
-
return self
|
50
|
-
|
51
|
-
def __exit__(self, exc_type, exc_val, exc_tb):
|
52
|
-
self.close()
|
53
|
-
|
54
|
-
def do_work(self):
|
55
|
-
# print (rank, name, 'start work')
|
56
|
-
while True:
|
57
|
-
method, args, kwargs = comm.scatter(None)
|
58
|
-
# print (rank, 'do work', method, args, kwargs)
|
59
|
-
if method == 'skip':
|
60
|
-
res = None
|
61
|
-
else:
|
62
|
-
res = getattr(self.object, method)
|
63
|
-
|
64
|
-
if hasattr(res, '__call__'):
|
65
|
-
res = res(*args, **kwargs)
|
66
|
-
# print (method, args, kwargs, res)
|
67
|
-
comm.gather(res)
|
68
|
-
if method == 'close':
|
69
|
-
if exit_mpi_on_close:
|
70
|
-
comm.gather(f'Exit, rank {rank}')
|
71
|
-
sys.exit(0)
|
72
|
-
else:
|
73
|
-
raise ChildProcessError('MPI worker done')
|
74
|
-
|
75
|
-
def __getattribute__(self, name):
|
76
|
-
if rank > 0:
|
77
|
-
return object.__getattribute__(self, name)
|
78
|
-
if name in MPIClassInterfaceAttributes:
|
79
|
-
return object.__getattribute__(self, name)
|
80
|
-
|
81
|
-
use_rank = self.use_rank
|
82
|
-
N = np.sum(use_rank)
|
83
|
-
|
84
|
-
def wrap(*args, **kwargs):
|
85
|
-
def get_input(i):
|
86
|
-
if use_rank[i]:
|
87
|
-
def get_arg(arg):
|
88
|
-
if isinstance(arg, list) and len(arg) == N:
|
89
|
-
return arg[np.where(use_rank)[0][i]]
|
90
|
-
else:
|
91
|
-
return arg
|
92
|
-
return (name, [get_arg(arg) for arg in args], {k: get_arg(v) for k, v in kwargs.items()})
|
93
|
-
else:
|
94
|
-
return ('skip', [], {})
|
95
|
-
inp = [get_input(i) for i in range(size)]
|
96
|
-
comm.scatter(inp, root=0)
|
97
|
-
if rank == 0:
|
98
|
-
method, args, kwargs = inp[0]
|
99
|
-
# print (rank, 'do work', method, args, kwargs)
|
100
|
-
if method == 'skip':
|
101
|
-
res = None
|
102
|
-
else:
|
103
|
-
res = getattr(self.object, method)
|
104
|
-
|
105
|
-
if hasattr(res, '__call__'):
|
106
|
-
res = res(*args, **kwargs)
|
107
|
-
|
108
|
-
# print (method, args, kwargs, res)
|
109
|
-
else:
|
110
|
-
res = None
|
111
|
-
|
112
|
-
res = [res for i, res in enumerate(comm.gather(res, root=0)) if use_rank[i]]
|
113
|
-
if isinstance(self, SubsetMPIClassInterface) and len(res) == 1:
|
114
|
-
return res[0]
|
115
|
-
return res
|
116
|
-
if hasattr(getattr(self.cls, name), '__call__'):
|
117
|
-
return wrap
|
118
|
-
else:
|
119
|
-
return wrap()
|
120
|
-
|
121
|
-
def __getitem__(self, slice):
|
122
|
-
use_rank = np.full_like(self.use_rank, False)
|
123
|
-
use_rank[slice] = True
|
124
|
-
return SubsetMPIClassInterface(self.cls, self.object, use_rank)
|
125
|
-
|
126
|
-
def close(self):
|
127
|
-
comm.scatter([('close', [], {}) for _ in range(size)], root=0)
|
128
|
-
(comm.gather(self.object.close(), root=0))
|
129
|
-
if exit_mpi_on_close:
|
130
|
-
(comm.gather(f'Exit, rank {rank}', root=0))
|
131
|
-
|
132
|
-
|
133
|
-
class SubsetMPIClassInterface(MPIClassInterface):
|
134
|
-
def __init__(self, cls, object, use_rank):
|
135
|
-
self.use_rank = use_rank
|
136
|
-
self.cls = cls
|
137
|
-
self.object = object
|
138
|
-
|
139
|
-
def __getitem__(self, slice):
|
140
|
-
l = np.arange(np.sum(self.use_rank))
|
141
|
-
if np.all(l == l[slice]):
|
142
|
-
return self
|
143
|
-
raise Exception('Cannot make subset of SubsetMPIClassInterface')
|
144
|
-
|
145
|
-
def close(self):
|
146
|
-
raise Exception('Cannot close SubsetMPIClassInterface. Please close all instances at once')
|
h2lib/utils.py
DELETED
@@ -1,183 +0,0 @@
|
|
1
|
-
import multiprocessing
|
2
|
-
import atexit
|
3
|
-
import numpy as np
|
4
|
-
from threading import Thread
|
5
|
-
from functools import wraps
|
6
|
-
import traceback
|
7
|
-
from _queue import Empty
|
8
|
-
import os
|
9
|
-
from contextlib import contextmanager
|
10
|
-
|
11
|
-
|
12
|
-
@contextmanager
|
13
|
-
def set_LD_LIBRARY_PATH():
|
14
|
-
_file__ = np.__file__
|
15
|
-
|
16
|
-
old = os.environ.get('LD_LIBRARY_PATH', "")
|
17
|
-
|
18
|
-
if '/lib/' in _file__: # pragma: no cover (only on linux)
|
19
|
-
lib_path = _file__[:_file__.index("/lib/") + 5]
|
20
|
-
os.environ['LD_LIBRARY_PATH'] = f'{lib_path}:{old}'
|
21
|
-
|
22
|
-
try:
|
23
|
-
yield
|
24
|
-
finally:
|
25
|
-
os.environ['LD_LIBRARY_PATH'] = old
|
26
|
-
|
27
|
-
|
28
|
-
def run(cls, inputQueue, outputQueue, cls_args, **kwargs):
|
29
|
-
o = cls(*cls_args, **kwargs)
|
30
|
-
while True:
|
31
|
-
method, args, kwargs = inputQueue.get()
|
32
|
-
# if method == "Sentinel":
|
33
|
-
# outputQueue.put(method)
|
34
|
-
# else:
|
35
|
-
# print(method, args, kwargs)
|
36
|
-
att = getattr(o, method)
|
37
|
-
if hasattr(att, '__call__'):
|
38
|
-
outputQueue.put(att(*args, **kwargs))
|
39
|
-
else:
|
40
|
-
outputQueue.put(att)
|
41
|
-
if method == 'close':
|
42
|
-
outputQueue.put('Exit process')
|
43
|
-
return
|
44
|
-
|
45
|
-
|
46
|
-
class ProcessClass():
|
47
|
-
cls = None
|
48
|
-
|
49
|
-
def __init__(self, cls, cls_attrs={}):
|
50
|
-
self.cls_attrs = cls_attrs
|
51
|
-
self.cls = cls
|
52
|
-
self.ctx = multiprocessing.get_context('spawn')
|
53
|
-
self.inputQueue = self.ctx.Queue()
|
54
|
-
self.outputQueue = self.ctx.Queue()
|
55
|
-
atexit.register(self.close)
|
56
|
-
self.closed = False
|
57
|
-
|
58
|
-
def __call__(self, *args, **kwargs):
|
59
|
-
kwargs.update({'cls': self.cls, 'inputQueue': self.inputQueue, 'outputQueue': self.outputQueue,
|
60
|
-
'cls_args': args})
|
61
|
-
s = 'vs_debug.py'
|
62
|
-
if s in "".join(traceback.format_stack()): # pragma: no cover
|
63
|
-
self.process = Thread(target=run, kwargs=kwargs) # use this to debug from Visual studio
|
64
|
-
else:
|
65
|
-
self.process = self.ctx.Process(target=run, kwargs=kwargs, daemon=True)
|
66
|
-
|
67
|
-
self.process.start()
|
68
|
-
return self
|
69
|
-
|
70
|
-
def __enter__(self):
|
71
|
-
return self
|
72
|
-
|
73
|
-
def __exit__(self, exc_type, exc_val, exc_tb):
|
74
|
-
self.close()
|
75
|
-
|
76
|
-
def __getattribute__(self, name):
|
77
|
-
try:
|
78
|
-
if name != 'cls_attrs' and name in self.cls_attrs:
|
79
|
-
raise AttributeError()
|
80
|
-
return object.__getattribute__(self, name)
|
81
|
-
except AttributeError:
|
82
|
-
if hasattr(self.cls, name):
|
83
|
-
if hasattr(getattr(self.cls, name), '__call__'):
|
84
|
-
@wraps(getattr(self.cls, name))
|
85
|
-
def wrap(*args, wait_for_result=True, **kwargs):
|
86
|
-
self.inputQueue.put((name, args, kwargs))
|
87
|
-
if wait_for_result:
|
88
|
-
return self.get_result()
|
89
|
-
return wrap
|
90
|
-
else:
|
91
|
-
self.inputQueue.put((name, (), {}))
|
92
|
-
return self.get_result()
|
93
|
-
else:
|
94
|
-
raise AttributeError(f"'{self.cls.__name__}' object has no attribute '{name}'")
|
95
|
-
|
96
|
-
def get_result(self):
|
97
|
-
while True:
|
98
|
-
if self.process.is_alive() or self.closed:
|
99
|
-
try:
|
100
|
-
return self.outputQueue.get(timeout=5)
|
101
|
-
except Empty:
|
102
|
-
pass # time out. Check process is alive and try again
|
103
|
-
# self.inputQueue.put(('Sentinel', (), {}))
|
104
|
-
# res_lst = []
|
105
|
-
# print('get result')
|
106
|
-
# while True:
|
107
|
-
# print('get')
|
108
|
-
# r = self.outputQueue.get()
|
109
|
-
# print(' got')
|
110
|
-
#
|
111
|
-
# if isinstance(r, str):
|
112
|
-
# if r == 'Exit process':
|
113
|
-
# return r
|
114
|
-
# elif r == 'Sentinel':
|
115
|
-
# return res_lst[0]
|
116
|
-
# else:
|
117
|
-
# res_lst.append(r)
|
118
|
-
# else:
|
119
|
-
# res_lst.append(r)
|
120
|
-
else:
|
121
|
-
raise Exception(f'{self.cls.__name__} process died')
|
122
|
-
|
123
|
-
def close(self, wait_for_result=False):
|
124
|
-
self.inputQueue.put(('close', [], {}))
|
125
|
-
self.process.join()
|
126
|
-
self.closed = True
|
127
|
-
|
128
|
-
|
129
|
-
class MultiProcessInterface():
|
130
|
-
def __init__(self, cls, args_lst):
|
131
|
-
self.cls = cls
|
132
|
-
self.obj_lst = [ProcessClass(cls)(*args) for args in args_lst]
|
133
|
-
|
134
|
-
def __getattribute__(self, name):
|
135
|
-
if name in ['obj_lst', '__class__']:
|
136
|
-
return object.__getattribute__(self, name)
|
137
|
-
|
138
|
-
a_lst = self.obj_lst
|
139
|
-
if hasattr(getattr(self.obj_lst[0], name), '__call__'):
|
140
|
-
def wrap(*args, **kwargs):
|
141
|
-
for i, o in enumerate(self.obj_lst):
|
142
|
-
def get_arg(arg):
|
143
|
-
if isinstance(arg, list) and len(arg) == len(a_lst):
|
144
|
-
return arg[i]
|
145
|
-
else:
|
146
|
-
return arg
|
147
|
-
a_args = [get_arg(arg) for arg in args]
|
148
|
-
a_kwargs = {k: get_arg(v) for k, v in kwargs.items()}
|
149
|
-
getattr(o, name)(*a_args, wait_for_result=False, **a_kwargs)
|
150
|
-
if isinstance(self, SubsetProcessWrapper) and len(self.obj_lst) == 1:
|
151
|
-
return self.obj_lst[0].get_result()
|
152
|
-
return [o.get_result() for o in self.obj_lst]
|
153
|
-
return wrap
|
154
|
-
else:
|
155
|
-
if isinstance(self, SubsetProcessWrapper) and len(self.obj_lst) == 1:
|
156
|
-
return getattr(self.obj_lst[0], name)
|
157
|
-
return [getattr(o, name) for o in self.obj_lst]
|
158
|
-
|
159
|
-
def __getitem__(self, slice):
|
160
|
-
lst = np.atleast_1d(np.array(self.obj_lst)[slice]).tolist()
|
161
|
-
return SubsetProcessWrapper(lst)
|
162
|
-
|
163
|
-
def __enter__(self):
|
164
|
-
return self
|
165
|
-
|
166
|
-
def __exit__(self, exc_type, exc_val, exc_tb):
|
167
|
-
self.close()
|
168
|
-
|
169
|
-
|
170
|
-
class SubsetProcessWrapper(MultiProcessInterface):
|
171
|
-
def __init__(self, obj_lst):
|
172
|
-
self.obj_lst = obj_lst
|
173
|
-
|
174
|
-
def __getitem__(self, slice):
|
175
|
-
if np.all(np.atleast_1d(self.obj_lst[slice]) == self.obj_lst):
|
176
|
-
return self
|
177
|
-
raise Exception('Cannot make subset of SubsetProcessWrapper')
|
178
|
-
|
179
|
-
def __getattribute__(self, name):
|
180
|
-
if name == 'close':
|
181
|
-
raise Exception("Cannot close SubsetProcessWrapper. Please close all instances at once")
|
182
|
-
|
183
|
-
return MultiProcessInterface.__getattribute__(self, name)
|
h2lib-13.0.703.dist-info/RECORD
DELETED
@@ -1,13 +0,0 @@
|
|
1
|
-
h2lib/HAWC2Lib.dll,sha256=RjH_EDSEz9oqOmtXppxuUog10vvdyL-WATIaEb0AqMI,29351424
|
2
|
-
h2lib/__init__.py,sha256=f3fO4I6IEFRM9LaV2O3w9Pioj3GPI8qRl7P5Tg5ONtE,528
|
3
|
-
h2lib/_h2lib.py,sha256=2mqulVzbZn2f_1szqD-uZTOmorXfwHgYip0KLeY7x0E,13897
|
4
|
-
h2lib/_version.py,sha256=l5YRIkdbaglY19rcFleugk6hNIPcavWNTU-8L1zWzTk,157
|
5
|
-
h2lib/dll_wrapper.py,sha256=_Y8482hVQIejw4Ty6UympOjdJlx68dINRa6JV--5f38,11979
|
6
|
-
h2lib/h2lib_signatures.py,sha256=FJYtgUomztX09PbscyKqdJ-kyvDHPE_qB0gNm8Egit0,14228
|
7
|
-
h2lib/mpi_utils.py,sha256=JIBiW06dD1LcfNtalXNrsMzQmrWBWn8QdM4e5jgUUwk,4990
|
8
|
-
h2lib/my_test_cls.py,sha256=t0QONPc2qJwm4dY5icZd22JDmAha9bqcGYV2AljxgeM,729
|
9
|
-
h2lib/utils.py,sha256=28peKufwJv3g-4N0ffG6TQIpJehsTkxTk_nMq7JD4X0,6529
|
10
|
-
h2lib-13.0.703.dist-info/METADATA,sha256=jVTi_3gnyFILTohC78PeA3YGNLE93Z9XOptgGNU5umI,634
|
11
|
-
h2lib-13.0.703.dist-info/WHEEL,sha256=82gZYSdmJ63dmEqVXfVfYjP-6F3ar13SybIjiy8AOec,102
|
12
|
-
h2lib-13.0.703.dist-info/top_level.txt,sha256=y_a-tUqphEZQ_0nsWSMaSb21P8Lsd8hUxUdE9g2Dcbk,6
|
13
|
-
h2lib-13.0.703.dist-info/RECORD,,
|
@@ -1 +0,0 @@
|
|
1
|
-
h2lib
|