pydefx 9.14.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- mpmcn.py +154 -0
- pydefx/__init__.py +32 -0
- pydefx/allpurposebuilder.py +56 -0
- pydefx/configuration.py +120 -0
- pydefx/defaultschemabuilder.py +55 -0
- pydefx/localbuilder.py +37 -0
- pydefx/localstudy.py +78 -0
- pydefx/multijobbuilder.py +33 -0
- pydefx/multijobstudy.py +84 -0
- pydefx/parameters.py +169 -0
- pydefx/plugins/jobexecutor.py +130 -0
- pydefx/plugins/lightexecutor.py +40 -0
- pydefx/plugins/localexecutor.py +81 -0
- pydefx/plugins/mainjob.py +56 -0
- pydefx/plugins/pointeval.py +39 -0
- pydefx/plugins/srunexecutor.py +87 -0
- pydefx/pyscript.py +110 -0
- pydefx/pystudy.py +327 -0
- pydefx/salome_proxy.py +88 -0
- pydefx/sample.py +263 -0
- pydefx/samplecsviterator.py +191 -0
- pydefx/samplecsvmanager.py +136 -0
- pydefx/schemas/idefix_pyschema.xml +106 -0
- pydefx/schemas/plugin.py +81 -0
- pydefx/slurmbuilder.py +33 -0
- pydefx/slurmstudy.py +77 -0
- pydefx/studyexception.py +41 -0
- pydefx/studyresult.py +54 -0
- pydefx-9.14.0.dist-info/METADATA +22 -0
- pydefx-9.14.0.dist-info/RECORD +45 -0
- pydefx-9.14.0.dist-info/WHEEL +4 -0
- salome/bin/salome/test/CTestTestfile.cmake +28 -0
- salome/bin/salome/test/cpp/CTestTestfile.cmake +36 -0
- salome/bin/salome/test/cpp/SampleTest +0 -0
- salome/bin/salome/test/cpp/StudyGeneralTest +0 -0
- salome/bin/salome/test/cpp/StudyRestartTest +0 -0
- salome/bin/salome/test/pyexample/CTestTestfile.cmake +25 -0
- salome/bin/salome/test/pyexample/insitu/insituiterator.py +48 -0
- salome/bin/salome/test/pyexample/insitu/insitumanager.py +46 -0
- salome/bin/salome/test/pyexample/runUnitTest.sh +26 -0
- salome/bin/salome/test/pyexample/test_default.py +79 -0
- salome/bin/salome/test/pyexample/test_insitu.py +60 -0
- salome/bin/salome/test/pyexample/test_mpmcn.py +40 -0
- salome/bin/salome/test/pyexample/test_prescript.py +52 -0
- salome/bin/salome/test/pyexample/test_ydefx_base.py +79 -0
pydefx/sample.py
ADDED
|
@@ -0,0 +1,263 @@
|
|
|
1
|
+
# -*- coding: utf-8 -*-
|
|
2
|
+
# Copyright (C) 2019-2024 EDF
|
|
3
|
+
#
|
|
4
|
+
# This library is free software; you can redistribute it and/or
|
|
5
|
+
# modify it under the terms of the GNU Lesser General Public
|
|
6
|
+
# License as published by the Free Software Foundation; either
|
|
7
|
+
# version 2.1 of the License, or (at your option) any later version.
|
|
8
|
+
#
|
|
9
|
+
# This library is distributed in the hope that it will be useful,
|
|
10
|
+
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
11
|
+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
|
12
|
+
# Lesser General Public License for more details.
|
|
13
|
+
#
|
|
14
|
+
# You should have received a copy of the GNU Lesser General Public
|
|
15
|
+
# License along with this library; if not, write to the Free Software
|
|
16
|
+
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
|
17
|
+
#
|
|
18
|
+
# See http://www.salome-platform.org/ or email : webmaster.salome@opencascade.com
|
|
19
|
+
#
|
|
20
|
+
import csv
|
|
21
|
+
from enum import Enum
|
|
22
|
+
|
|
23
|
+
class SampleException(Exception):
|
|
24
|
+
pass
|
|
25
|
+
|
|
26
|
+
class PointState(Enum):
|
|
27
|
+
"""
|
|
28
|
+
Is the point evaluated?
|
|
29
|
+
"""
|
|
30
|
+
NO = 1
|
|
31
|
+
YES = 2
|
|
32
|
+
ERROR = 3
|
|
33
|
+
|
|
34
|
+
class Sample:
|
|
35
|
+
def __init__(self, input_names, output_names):
|
|
36
|
+
"""
|
|
37
|
+
input_names : the names and the order of input variables
|
|
38
|
+
output_names : the names and the order of output variables
|
|
39
|
+
"""
|
|
40
|
+
if len(input_names) < 1 :
|
|
41
|
+
SampleException("A sample should have at least one input variable.")
|
|
42
|
+
self._input_names = input_names
|
|
43
|
+
self._output_names = output_names
|
|
44
|
+
self._input={}
|
|
45
|
+
self._output={} # {output_name:[output_values]}
|
|
46
|
+
self._messages=[] # list. Possible values in the list:
|
|
47
|
+
# None = point not evaluated,
|
|
48
|
+
# "" = results available, no error
|
|
49
|
+
# "any string"= error message for that index.
|
|
50
|
+
|
|
51
|
+
def __iter__(self):
|
|
52
|
+
"""
|
|
53
|
+
Iterator over input points.
|
|
54
|
+
"""
|
|
55
|
+
return Dico_iter(self._input)
|
|
56
|
+
|
|
57
|
+
def inputIterator(self):
|
|
58
|
+
"""
|
|
59
|
+
Iterate over input points.
|
|
60
|
+
"""
|
|
61
|
+
return Dico_iter(self._input)
|
|
62
|
+
|
|
63
|
+
def setInputValues(self, inputDict):
|
|
64
|
+
"""
|
|
65
|
+
Set the points to be evaluated.
|
|
66
|
+
inputDict is a dictionary with:
|
|
67
|
+
key : name of an input parameter
|
|
68
|
+
value : list of values for the key.
|
|
69
|
+
The lists of values should have the same size for all the keys.
|
|
70
|
+
Exemple :
|
|
71
|
+
mySample.setInputValues({ "x":[1,2,3], "y":[10,20,30]})
|
|
72
|
+
"""
|
|
73
|
+
if len(inputDict) != len(self._input_names):
|
|
74
|
+
raise SampleException("Incorrect number of keys!")
|
|
75
|
+
size = None
|
|
76
|
+
for name in self._input_names:
|
|
77
|
+
if size is None:
|
|
78
|
+
size = len(inputDict[name])
|
|
79
|
+
else:
|
|
80
|
+
if size != len(inputDict[name]):
|
|
81
|
+
raise SampleException("Inconsistency in input variable sizes!")
|
|
82
|
+
self._input = inputDict
|
|
83
|
+
# Fill all results with None
|
|
84
|
+
self._messages = [ None for i in range(size)]
|
|
85
|
+
self._output.clear()
|
|
86
|
+
for name in self._output_names :
|
|
87
|
+
self._output[name] = [ None for i in range(size)]
|
|
88
|
+
|
|
89
|
+
def addResult(self, index, out_values, message):
|
|
90
|
+
"""
|
|
91
|
+
Set the resut for a point.
|
|
92
|
+
Parameters :
|
|
93
|
+
index : index of the point to be set
|
|
94
|
+
out_values : dictionary with :
|
|
95
|
+
key : output parameter name
|
|
96
|
+
value : value to be set for this index
|
|
97
|
+
message : error message. An empty string means no error. Any other string
|
|
98
|
+
contains the error message.
|
|
99
|
+
If the message is not an empty string, out_values is ignored and all the
|
|
100
|
+
results for this point are set to None as the point is considered to be in
|
|
101
|
+
error.
|
|
102
|
+
"""
|
|
103
|
+
if message is None:
|
|
104
|
+
message = ""
|
|
105
|
+
if message :
|
|
106
|
+
# case of error
|
|
107
|
+
for name in self._output_names :
|
|
108
|
+
self._output[name][index] = None
|
|
109
|
+
else:
|
|
110
|
+
for name in self._output_names :
|
|
111
|
+
self._output[name][index] = out_values[name]
|
|
112
|
+
|
|
113
|
+
self._messages[index] = message
|
|
114
|
+
|
|
115
|
+
def getPointState(self, index):
|
|
116
|
+
"""
|
|
117
|
+
Computation state of the point number index.
|
|
118
|
+
Possible states returned :
|
|
119
|
+
PointState.NO : The results are not available yet for this point.
|
|
120
|
+
The point has not been evaluated yet.
|
|
121
|
+
PointState.ERROR : The point evaluation is finished but it failed.
|
|
122
|
+
There are no values for the output parameters, but there
|
|
123
|
+
is a message containing the error.
|
|
124
|
+
PointState.YES : The results are available for this point.
|
|
125
|
+
"""
|
|
126
|
+
message = self._messages[index]
|
|
127
|
+
if message is None:
|
|
128
|
+
return PointState.NO
|
|
129
|
+
elif message:
|
|
130
|
+
return PointState.ERROR
|
|
131
|
+
else:
|
|
132
|
+
return PointState.YES
|
|
133
|
+
|
|
134
|
+
def findFirstId(self, in_values):
|
|
135
|
+
"""
|
|
136
|
+
Find the index of the first point in the sample which contains exactly the
|
|
137
|
+
same input values as in in_values.
|
|
138
|
+
in_values is a dictionary with {name : value} of input variables in one
|
|
139
|
+
point.
|
|
140
|
+
"""
|
|
141
|
+
if self._input_names is None or len(self._input_names) == 0 :
|
|
142
|
+
raise SampleException("List of input variables not defined.")
|
|
143
|
+
firstName = self._input_names[0]
|
|
144
|
+
maxSize = len(self._input[firstName])
|
|
145
|
+
curId = 0
|
|
146
|
+
foundId = -1
|
|
147
|
+
for curId in range(maxSize):
|
|
148
|
+
if self.isValidId(self, curId, in_values) :
|
|
149
|
+
foundId = curId
|
|
150
|
+
break
|
|
151
|
+
return foundId
|
|
152
|
+
|
|
153
|
+
def isValidId(self, idToCheck, inputToCheck):
|
|
154
|
+
"""
|
|
155
|
+
Verify the input values at the idToCheck position are exactly the same as
|
|
156
|
+
the values in inputToCheck. Return True or False.
|
|
157
|
+
"""
|
|
158
|
+
ok = True
|
|
159
|
+
try:
|
|
160
|
+
for name in self._input_names:
|
|
161
|
+
if self._input[name][idToCheck] != inputToCheck[name] :
|
|
162
|
+
ok = False
|
|
163
|
+
break
|
|
164
|
+
except:
|
|
165
|
+
ok = False
|
|
166
|
+
return ok
|
|
167
|
+
|
|
168
|
+
def checkId(self, idToCheck, inputToCheck):
|
|
169
|
+
"""
|
|
170
|
+
Verify the input values at the idToCheck position are exactly the same as
|
|
171
|
+
the values in inputToCheck. Raise SampleException if it is not the case.
|
|
172
|
+
"""
|
|
173
|
+
for name in self._input_names:
|
|
174
|
+
if not name in self._input:
|
|
175
|
+
raise SampleException(
|
|
176
|
+
"Variable name {} expected and not found ".format(name))
|
|
177
|
+
if self._input[name][idToCheck] != inputToCheck[name] :
|
|
178
|
+
raise SampleException(
|
|
179
|
+
"Expected value for variable {} is {} and found {}".format(name,
|
|
180
|
+
self._input[name][idToCheck],
|
|
181
|
+
inputToCheck[name]))
|
|
182
|
+
|
|
183
|
+
def getInputNames(self):
|
|
184
|
+
"""
|
|
185
|
+
Return the list of input parameter names.
|
|
186
|
+
"""
|
|
187
|
+
return self._input_names
|
|
188
|
+
|
|
189
|
+
def getOutputNames(self):
|
|
190
|
+
"""
|
|
191
|
+
Return the list of output parameter names.
|
|
192
|
+
"""
|
|
193
|
+
return self._output_names
|
|
194
|
+
|
|
195
|
+
def progressRate(self):
|
|
196
|
+
"""
|
|
197
|
+
Return 1.0 - unevaluatedPoints / numberOfPoints
|
|
198
|
+
"""
|
|
199
|
+
numberOfPoints = len(self._messages)
|
|
200
|
+
unevaluatedPoints = self._messages.count(None)
|
|
201
|
+
result = 0.0
|
|
202
|
+
if(numberOfPoints > 0):
|
|
203
|
+
result = 1.0 - unevaluatedPoints / numberOfPoints
|
|
204
|
+
return result
|
|
205
|
+
|
|
206
|
+
def getInput(self, name):
|
|
207
|
+
"""
|
|
208
|
+
Get the list of values for an input parameter.
|
|
209
|
+
"""
|
|
210
|
+
return self._input[name]
|
|
211
|
+
|
|
212
|
+
def getOutput(self, name):
|
|
213
|
+
"""
|
|
214
|
+
Get the list of values for an output parameter.
|
|
215
|
+
"""
|
|
216
|
+
return self._output[name]
|
|
217
|
+
|
|
218
|
+
def getMessages(self):
|
|
219
|
+
"""
|
|
220
|
+
Get the list of messages where for any point we can have :
|
|
221
|
+
None - the point has not been evaluated yet.
|
|
222
|
+
"" (empty string) - evaluation ok, no error.
|
|
223
|
+
"any string" - error message.
|
|
224
|
+
"""
|
|
225
|
+
return self._messages
|
|
226
|
+
|
|
227
|
+
def __str__(self):
|
|
228
|
+
result = ""
|
|
229
|
+
for name in self._input_names:
|
|
230
|
+
result += name+","
|
|
231
|
+
for name in self._output_names:
|
|
232
|
+
result += name+","
|
|
233
|
+
result += "messages\n"
|
|
234
|
+
for i in range(len(self._messages)):
|
|
235
|
+
for name in self._input_names:
|
|
236
|
+
result += repr(self._input[name][i]) +","
|
|
237
|
+
for name in self._output_names:
|
|
238
|
+
result += repr(self._output[name][i]) +","
|
|
239
|
+
result += str(self._messages[i])+"\n"
|
|
240
|
+
return result
|
|
241
|
+
|
|
242
|
+
class Dico_iter:
|
|
243
|
+
"""
|
|
244
|
+
>>> si=Dico_iter({"x":[1,2,3], "y":["a","b","c"]})
|
|
245
|
+
>>> for i in si:
|
|
246
|
+
>>> print(i)
|
|
247
|
+
{'x': 1, 'y': 'a'}
|
|
248
|
+
{'x': 2, 'y': 'b'}
|
|
249
|
+
{'x': 3, 'y': 'c'}
|
|
250
|
+
"""
|
|
251
|
+
def __init__(self, s):
|
|
252
|
+
self.iters={}
|
|
253
|
+
for k, v in s.items():
|
|
254
|
+
self.iters[k] = iter(v)
|
|
255
|
+
|
|
256
|
+
def __iter__(self):
|
|
257
|
+
return self
|
|
258
|
+
|
|
259
|
+
def __next__(self):
|
|
260
|
+
result = {}
|
|
261
|
+
for k,v in self.iters.items():
|
|
262
|
+
result[k] = next(v)
|
|
263
|
+
return result
|
|
@@ -0,0 +1,191 @@
|
|
|
1
|
+
# Copyright (C) 2019-2024 EDF
|
|
2
|
+
#
|
|
3
|
+
# This library is free software; you can redistribute it and/or
|
|
4
|
+
# modify it under the terms of the GNU Lesser General Public
|
|
5
|
+
# License as published by the Free Software Foundation; either
|
|
6
|
+
# version 2.1 of the License, or (at your option) any later version.
|
|
7
|
+
#
|
|
8
|
+
# This library is distributed in the hope that it will be useful,
|
|
9
|
+
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
10
|
+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
|
11
|
+
# Lesser General Public License for more details.
|
|
12
|
+
#
|
|
13
|
+
# You should have received a copy of the GNU Lesser General Public
|
|
14
|
+
# License along with this library; if not, write to the Free Software
|
|
15
|
+
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
|
16
|
+
#
|
|
17
|
+
# See http://www.salome-platform.org/ or email : webmaster.salome@opencascade.com
|
|
18
|
+
#
|
|
19
|
+
import csv
|
|
20
|
+
import numbers
|
|
21
|
+
import pickle
|
|
22
|
+
import os
|
|
23
|
+
|
|
24
|
+
class SampleIterator:
|
|
25
|
+
"""
|
|
26
|
+
Iterator used to iterate over the input values of a sample, adding an order
|
|
27
|
+
number. The order number is the id you get as the first parameter of the
|
|
28
|
+
function addResult.
|
|
29
|
+
"""
|
|
30
|
+
DATAFILE = "idefixdata.csv"
|
|
31
|
+
OUTPUTNAMESFILE = "idefixoutputnames.csv"
|
|
32
|
+
RESULTDIR = "idefixresult" # directory which contains all the result files
|
|
33
|
+
RESULTFILE = "idefixresult.csv" # main result file - values for every point
|
|
34
|
+
GLOBALFILE = "idefixglobal" # global result - one value for the whole simulation
|
|
35
|
+
ERRORCOLUMN = "idefix_error"
|
|
36
|
+
IDCOLUMN ="idefix_id"
|
|
37
|
+
ESCAPE_CHAR = "@" # prefix a value that needs particular save/load procedure
|
|
38
|
+
PICK_CHAR = "p" # @p : csv value saved in another file using pickle
|
|
39
|
+
|
|
40
|
+
def __init__(self, directory=None):
|
|
41
|
+
if directory:
|
|
42
|
+
datapath = os.path.join(directory, SampleIterator.DATAFILE)
|
|
43
|
+
outputnamespath = os.path.join(directory, SampleIterator.OUTPUTNAMESFILE)
|
|
44
|
+
self.result_directory = os.path.join(directory, SampleIterator.RESULTDIR)
|
|
45
|
+
self.directory = directory
|
|
46
|
+
else:
|
|
47
|
+
datapath = SampleIterator.DATAFILE
|
|
48
|
+
outputnamespath = SampleIterator.OUTPUTNAMESFILE
|
|
49
|
+
self.result_directory = SampleIterator.RESULTDIR
|
|
50
|
+
self.directory = None
|
|
51
|
+
self.result_file = None
|
|
52
|
+
self.datafile = open(datapath, newline='')
|
|
53
|
+
self.data = csv.DictReader(self.datafile, quoting=csv.QUOTE_NONNUMERIC)
|
|
54
|
+
self.inputnames = self.data.fieldnames
|
|
55
|
+
self.outputnames = _loadOutputNames(outputnamespath)
|
|
56
|
+
self.iterNb = -1
|
|
57
|
+
|
|
58
|
+
def __next__(self):
|
|
59
|
+
self.iterNb += 1
|
|
60
|
+
return self.iterNb, next(self.data)
|
|
61
|
+
|
|
62
|
+
def __iter__(self):
|
|
63
|
+
return self
|
|
64
|
+
|
|
65
|
+
def writeHeaders(self):
|
|
66
|
+
"""
|
|
67
|
+
This function can be called before the first call to addResult in order to
|
|
68
|
+
write the names of the parameters in the result file.
|
|
69
|
+
"""
|
|
70
|
+
if self.directory:
|
|
71
|
+
outputnamespath = os.path.join(self.directory,
|
|
72
|
+
SampleIterator.OUTPUTNAMESFILE)
|
|
73
|
+
else:
|
|
74
|
+
outputnamespath = SampleIterator.OUTPUTNAMESFILE
|
|
75
|
+
os.makedirs(self.result_directory, exist_ok=True)
|
|
76
|
+
resultpath = os.path.join(self.result_directory, SampleIterator.RESULTFILE)
|
|
77
|
+
result_columns = [SampleIterator.IDCOLUMN]
|
|
78
|
+
result_columns.extend(self.inputnames)
|
|
79
|
+
result_columns.extend(self.outputnames)
|
|
80
|
+
result_columns.append(SampleIterator.ERRORCOLUMN)
|
|
81
|
+
self.result_file = open(resultpath, 'w', newline='')
|
|
82
|
+
self.result_csv = csv.DictWriter( self.result_file,
|
|
83
|
+
fieldnames=result_columns,
|
|
84
|
+
quoting=csv.QUOTE_NONNUMERIC )
|
|
85
|
+
self.result_csv.writeheader()
|
|
86
|
+
self.result_file.flush()
|
|
87
|
+
|
|
88
|
+
def addResult(self, currentId, currentInput, currentOutput, currentError):
|
|
89
|
+
"""
|
|
90
|
+
You need to call writeHeaders before the first call of this function.
|
|
91
|
+
currentId : int value
|
|
92
|
+
currentInput : dictionary {"input name":value}
|
|
93
|
+
currentOutput : result returned by _exec. Can be a tuple, a simple value or
|
|
94
|
+
None in case of error.
|
|
95
|
+
currentError : string or None if no error
|
|
96
|
+
"""
|
|
97
|
+
currentRecord = {}
|
|
98
|
+
currentRecord[SampleIterator.IDCOLUMN] = currentId
|
|
99
|
+
for name in self.inputnames:
|
|
100
|
+
currentRecord[name] = currentInput[name]
|
|
101
|
+
if currentError is None:
|
|
102
|
+
if len(self.outputnames) == 1 :
|
|
103
|
+
outputname = self.outputnames[0]
|
|
104
|
+
currentRecord[outputname] = _codeOutput(currentOutput,
|
|
105
|
+
currentId,
|
|
106
|
+
outputname,
|
|
107
|
+
self.directory)
|
|
108
|
+
elif len(self.outputnames) > 1 :
|
|
109
|
+
outputIter = iter(currentOutput)
|
|
110
|
+
for name in self.outputnames:
|
|
111
|
+
currentRecord[name] = _codeOutput(next(outputIter),
|
|
112
|
+
currentId,
|
|
113
|
+
name,
|
|
114
|
+
self.directory)
|
|
115
|
+
else:
|
|
116
|
+
for name in self.outputnames:
|
|
117
|
+
currentRecord[name] = None
|
|
118
|
+
currentRecord[SampleIterator.ERRORCOLUMN] = currentError
|
|
119
|
+
self.result_csv.writerow(currentRecord)
|
|
120
|
+
self.result_file.flush()
|
|
121
|
+
|
|
122
|
+
def terminate(self):
|
|
123
|
+
"""
|
|
124
|
+
Call this function at the end of the simulation in order to close every
|
|
125
|
+
open files.
|
|
126
|
+
"""
|
|
127
|
+
if not self.datafile is None:
|
|
128
|
+
self.datafile.close()
|
|
129
|
+
self.datafile = None
|
|
130
|
+
if not self.result_file is None:
|
|
131
|
+
self.result_file.close()
|
|
132
|
+
self.result_file = None
|
|
133
|
+
|
|
134
|
+
# Private functions
|
|
135
|
+
def _loadOutputNames(filepath):
|
|
136
|
+
outputnames = []
|
|
137
|
+
with open(filepath, "r") as namesfile:
|
|
138
|
+
for line in namesfile:
|
|
139
|
+
line = line.rstrip() # remove whitespaces at the end
|
|
140
|
+
outputnames.append(line)
|
|
141
|
+
return outputnames
|
|
142
|
+
|
|
143
|
+
# Read and write results (output parameters)
|
|
144
|
+
def _codeOutput(value, currentId, name, directory=None):
|
|
145
|
+
"""
|
|
146
|
+
Define how a value should be saved.
|
|
147
|
+
value: object to be saved - value of a parameter
|
|
148
|
+
currentId: number of the current line (current point).
|
|
149
|
+
name: name of the parameter (name of the column in the csv file).
|
|
150
|
+
return: string to be saved in the csv file.
|
|
151
|
+
"""
|
|
152
|
+
res = None
|
|
153
|
+
if isinstance(value, numbers.Number):
|
|
154
|
+
res = value
|
|
155
|
+
elif isinstance(value, str):
|
|
156
|
+
res = value
|
|
157
|
+
if res[0:1] == SampleIterator.ESCAPE_CHAR :
|
|
158
|
+
res = SampleIterator.ESCAPE_CHAR + res
|
|
159
|
+
else:
|
|
160
|
+
file_name = "idefixresult-{}-{}.pick".format(name, currentId)
|
|
161
|
+
res = SampleIterator.ESCAPE_CHAR + SampleIterator.PICK_CHAR + file_name
|
|
162
|
+
file_path = os.path.join(SampleIterator.RESULTDIR, file_name)
|
|
163
|
+
if directory :
|
|
164
|
+
file_path = os.path.join(directory, file_path)
|
|
165
|
+
with open(file_path, "wb") as f:
|
|
166
|
+
pickle.dump(value, f)
|
|
167
|
+
return res
|
|
168
|
+
|
|
169
|
+
def _decodeOutput(obj, resultdir):
|
|
170
|
+
"""
|
|
171
|
+
Decode a value read from the csv file.
|
|
172
|
+
obj: object to decode (string or number).
|
|
173
|
+
resultdir : directory which contains the result files
|
|
174
|
+
return: decoded object.
|
|
175
|
+
"""
|
|
176
|
+
res = None
|
|
177
|
+
if isinstance(obj, numbers.Number):
|
|
178
|
+
res = obj
|
|
179
|
+
elif isinstance(obj, str):
|
|
180
|
+
res = obj
|
|
181
|
+
if res[0:1] == SampleIterator.ESCAPE_CHAR :
|
|
182
|
+
res = res[1:]
|
|
183
|
+
if res[0:1] == SampleIterator.ESCAPE_CHAR :# obj = @@string begins with@
|
|
184
|
+
pass
|
|
185
|
+
elif res[0:1] == SampleIterator.PICK_CHAR:# obj = @pidefixresult-x-1.pick
|
|
186
|
+
file_path = os.path.join(resultdir, res[1:])
|
|
187
|
+
with open(file_path, "rb") as f:
|
|
188
|
+
res = pickle.load(f)
|
|
189
|
+
else:
|
|
190
|
+
raise Exception("Unknown escape value:" + obj)
|
|
191
|
+
return res
|
|
@@ -0,0 +1,136 @@
|
|
|
1
|
+
# Copyright (C) 2019-2024 EDF
|
|
2
|
+
#
|
|
3
|
+
# This library is free software; you can redistribute it and/or
|
|
4
|
+
# modify it under the terms of the GNU Lesser General Public
|
|
5
|
+
# License as published by the Free Software Foundation; either
|
|
6
|
+
# version 2.1 of the License, or (at your option) any later version.
|
|
7
|
+
#
|
|
8
|
+
# This library is distributed in the hope that it will be useful,
|
|
9
|
+
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
10
|
+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
|
11
|
+
# Lesser General Public License for more details.
|
|
12
|
+
#
|
|
13
|
+
# You should have received a copy of the GNU Lesser General Public
|
|
14
|
+
# License along with this library; if not, write to the Free Software
|
|
15
|
+
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
|
16
|
+
#
|
|
17
|
+
# See http://www.salome-platform.org/ or email : webmaster.salome@opencascade.com
|
|
18
|
+
#
|
|
19
|
+
import csv
|
|
20
|
+
import inspect
|
|
21
|
+
import os
|
|
22
|
+
import pathlib
|
|
23
|
+
from . import sample
|
|
24
|
+
from . import samplecsviterator
|
|
25
|
+
SampleIterator = samplecsviterator.SampleIterator
|
|
26
|
+
|
|
27
|
+
class SampleManager:
|
|
28
|
+
"""
|
|
29
|
+
The SampleManager is used by the study for reading and writing a sample from
|
|
30
|
+
and to the file system. This SampleManager uses the csv format.
|
|
31
|
+
The following services are needed by the study:
|
|
32
|
+
- write the sample on the local file system (prepareRun).
|
|
33
|
+
- know what files were written in order to copy them on the remote file system
|
|
34
|
+
(return value of prepareRun).
|
|
35
|
+
- know what files contain the result in order to bring them back from the
|
|
36
|
+
remote file system to the local one (getResultFileName).
|
|
37
|
+
- load the results from the local file system to a sample (loadResult).
|
|
38
|
+
- restore a sample from a local directory when you want to recover a job
|
|
39
|
+
launched in a previous session.
|
|
40
|
+
- the name of the module which contains the class SampleIterator in order to
|
|
41
|
+
iterate over the input values of the sample (getModuleName).
|
|
42
|
+
This name is written by the study in a configuration file and it is used by
|
|
43
|
+
the optimizer loop plugin.
|
|
44
|
+
"""
|
|
45
|
+
def __init__(self):
|
|
46
|
+
pass
|
|
47
|
+
|
|
48
|
+
# Functions used by the study
|
|
49
|
+
def prepareRun(self, sample, directory):
|
|
50
|
+
"""
|
|
51
|
+
Create a dump of the sample in the given directory.
|
|
52
|
+
sample: Sample object.
|
|
53
|
+
directory: path to a local working directory where all the working files are
|
|
54
|
+
copied. This directory should be already created.
|
|
55
|
+
Return a list of files to add to the input files list of the job.
|
|
56
|
+
"""
|
|
57
|
+
datapath = os.path.join(directory, SampleIterator.DATAFILE)
|
|
58
|
+
with open(datapath, 'w', newline='') as csvfile:
|
|
59
|
+
writer = csv.DictWriter(csvfile,
|
|
60
|
+
fieldnames=sample.getInputNames(),
|
|
61
|
+
quoting=csv.QUOTE_NONNUMERIC )
|
|
62
|
+
writer.writeheader()
|
|
63
|
+
writer.writerows(sample.inputIterator())
|
|
64
|
+
|
|
65
|
+
outnamespath = os.path.join(directory, SampleIterator.OUTPUTNAMESFILE)
|
|
66
|
+
with open(outnamespath, 'w') as outputfile:
|
|
67
|
+
for v in sample.getOutputNames():
|
|
68
|
+
outputfile.write(v+'\n')
|
|
69
|
+
filename = inspect.getframeinfo(inspect.currentframe()).filename
|
|
70
|
+
install_directory = pathlib.Path(filename).resolve().parent
|
|
71
|
+
iteratorFile = os.path.join(install_directory, "samplecsviterator.py")
|
|
72
|
+
return [datapath,
|
|
73
|
+
outnamespath,
|
|
74
|
+
iteratorFile
|
|
75
|
+
]
|
|
76
|
+
|
|
77
|
+
def loadResult(self, sample, directory):
|
|
78
|
+
"""
|
|
79
|
+
The directory should contain a RESULTDIR directory with the result files.
|
|
80
|
+
The results are loaded into the sample.
|
|
81
|
+
Return the global result of the study which can be used by an insitu
|
|
82
|
+
computation.
|
|
83
|
+
"""
|
|
84
|
+
resultdir = os.path.join(directory, SampleIterator.RESULTDIR)
|
|
85
|
+
datapath = os.path.join(resultdir, SampleIterator.RESULTFILE)
|
|
86
|
+
with open(datapath, newline='') as datafile:
|
|
87
|
+
data = csv.DictReader(datafile, quoting=csv.QUOTE_NONNUMERIC)
|
|
88
|
+
for elt in data:
|
|
89
|
+
index = int(elt[SampleIterator.IDCOLUMN]) # float to int
|
|
90
|
+
input_vals = {}
|
|
91
|
+
for name in sample.getInputNames():
|
|
92
|
+
input_vals[name] = elt[name]
|
|
93
|
+
output_vals = {}
|
|
94
|
+
for name in sample.getOutputNames():
|
|
95
|
+
output_vals[name] = samplecsviterator._decodeOutput(elt[name],
|
|
96
|
+
resultdir)
|
|
97
|
+
try:
|
|
98
|
+
sample.checkId(index, input_vals)
|
|
99
|
+
except Exception as err:
|
|
100
|
+
extraInfo = "Error on processing file {} index number {}:".format(
|
|
101
|
+
datapath, str(index))
|
|
102
|
+
raise Exception(extraInfo + str(err))
|
|
103
|
+
sample.addResult(index, output_vals, elt[SampleIterator.ERRORCOLUMN])
|
|
104
|
+
return sample
|
|
105
|
+
|
|
106
|
+
def restoreSample(self, directory):
|
|
107
|
+
""" The directory should contain the files created by prepareRun. A new
|
|
108
|
+
sample object is created and returned from those files.
|
|
109
|
+
This function is used to recover a previous run.
|
|
110
|
+
"""
|
|
111
|
+
sampleIt = SampleIterator(directory)
|
|
112
|
+
inputvalues = {}
|
|
113
|
+
for name in sampleIt.inputnames:
|
|
114
|
+
inputvalues[name] = []
|
|
115
|
+
for newid, values in sampleIt:
|
|
116
|
+
for name in sampleIt.inputnames:
|
|
117
|
+
inputvalues[name].append(values[name])
|
|
118
|
+
|
|
119
|
+
result = sample.Sample(sampleIt.inputnames, sampleIt.outputnames)
|
|
120
|
+
result.setInputValues(inputvalues)
|
|
121
|
+
sampleIt.terminate()
|
|
122
|
+
return result
|
|
123
|
+
|
|
124
|
+
def getModuleName(self):
|
|
125
|
+
"""
|
|
126
|
+
Return the module name which contains the class SampleIterator.
|
|
127
|
+
"""
|
|
128
|
+
return "samplecsviterator"
|
|
129
|
+
|
|
130
|
+
def getResultFileName(self):
|
|
131
|
+
"""
|
|
132
|
+
Name of the file or directory which contains the result and needs to be
|
|
133
|
+
copied from the remote computer.
|
|
134
|
+
"""
|
|
135
|
+
return SampleIterator.RESULTDIR
|
|
136
|
+
|
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
<?xml version='1.0' encoding='iso-8859-1' ?>
|
|
2
|
+
<proc name="newSchema_1">
|
|
3
|
+
<type name="string" kind="string"/>
|
|
4
|
+
<struct name="Engines/dataref">
|
|
5
|
+
<member name="ref" type="string"/>
|
|
6
|
+
</struct>
|
|
7
|
+
<type name="bool" kind="bool"/>
|
|
8
|
+
<sequence name="boolvec" content="bool"/>
|
|
9
|
+
<type name="double" kind="double"/>
|
|
10
|
+
<sequence name="dblevec" content="double"/>
|
|
11
|
+
<objref name="file" id="file"/>
|
|
12
|
+
<type name="int" kind="int"/>
|
|
13
|
+
<sequence name="intvec" content="int"/>
|
|
14
|
+
<struct name="stringpair">
|
|
15
|
+
<member name="name" type="string"/>
|
|
16
|
+
<member name="value" type="string"/>
|
|
17
|
+
</struct>
|
|
18
|
+
<sequence name="propvec" content="stringpair"/>
|
|
19
|
+
<objref name="pyobj" id="python:obj:1.0"/>
|
|
20
|
+
<sequence name="seqboolvec" content="boolvec"/>
|
|
21
|
+
<sequence name="seqdblevec" content="dblevec"/>
|
|
22
|
+
<sequence name="seqintvec" content="intvec"/>
|
|
23
|
+
<sequence name="seqpyobj" content="pyobj"/>
|
|
24
|
+
<sequence name="stringvec" content="string"/>
|
|
25
|
+
<sequence name="seqstringvec" content="stringvec"/>
|
|
26
|
+
<container name="DefaultContainer">
|
|
27
|
+
<property name="container_kind" value="Salome"/>
|
|
28
|
+
<property name="attached_on_cloning" value="0"/>
|
|
29
|
+
<property name="container_name" value="FactoryServer"/>
|
|
30
|
+
<property name="name" value="localhost"/>
|
|
31
|
+
</container>
|
|
32
|
+
<container name="container0">
|
|
33
|
+
<property name="container_kind" value="Salome"/>
|
|
34
|
+
<property name="attached_on_cloning" value="0"/>
|
|
35
|
+
</container>
|
|
36
|
+
<optimizer name="OptimizerLoop0" nbranch="4" loopWeight="-1" lib="plugin.py" entry="myalgosync">
|
|
37
|
+
<remote name="Solver" elementaryWeight="-1">
|
|
38
|
+
<script><code><![CDATA[import pickle
|
|
39
|
+
import traceback
|
|
40
|
+
import importlib
|
|
41
|
+
inputvals=pickle.loads(i0.encode())
|
|
42
|
+
idefixstudy=importlib.import_module(studymodule)
|
|
43
|
+
error=None
|
|
44
|
+
result=None
|
|
45
|
+
try:
|
|
46
|
+
result=idefixstudy._exec(**inputvals)
|
|
47
|
+
except Exception as e:
|
|
48
|
+
error=str(e)
|
|
49
|
+
if not error :
|
|
50
|
+
error = "Exception " + repr(e)
|
|
51
|
+
traceback.print_exc()
|
|
52
|
+
o0=pickle.dumps((error, result), protocol=0).decode()
|
|
53
|
+
]]></code></script>
|
|
54
|
+
<load container="container0"/>
|
|
55
|
+
<inport name="i0" type="string"/>
|
|
56
|
+
<inport name="studymodule" type="string"/>
|
|
57
|
+
<outport name="o0" type="string"/>
|
|
58
|
+
</remote>
|
|
59
|
+
</optimizer>
|
|
60
|
+
<inline name="Initialisation">
|
|
61
|
+
<script><code><![CDATA[import json
|
|
62
|
+
|
|
63
|
+
with open("idefixconfig.json", "r") as f:
|
|
64
|
+
config = json.load(f)
|
|
65
|
+
nbbranches=config["nbbranches"]
|
|
66
|
+
studymodule=config["studymodule"]
|
|
67
|
+
|
|
68
|
+
prescript = None
|
|
69
|
+
try:
|
|
70
|
+
with open("idefix_prescript.py") as f:
|
|
71
|
+
prescript = f.read()
|
|
72
|
+
except FileNotFoundError:
|
|
73
|
+
pass
|
|
74
|
+
if not prescript is None:
|
|
75
|
+
exec(prescript)
|
|
76
|
+
]]></code></script>
|
|
77
|
+
<load container="DefaultContainer"/>
|
|
78
|
+
<outport name="nbbranches" type="int"/>
|
|
79
|
+
<outport name="studymodule" type="string"/>
|
|
80
|
+
</inline>
|
|
81
|
+
<control> <fromnode>Initialisation</fromnode> <tonode>OptimizerLoop0</tonode> </control>
|
|
82
|
+
<datalink control="false">
|
|
83
|
+
<fromnode>OptimizerLoop0</fromnode> <fromport>evalSamples</fromport>
|
|
84
|
+
<tonode>OptimizerLoop0.Solver</tonode> <toport>i0</toport>
|
|
85
|
+
</datalink>
|
|
86
|
+
<datalink control="false">
|
|
87
|
+
<fromnode>Initialisation</fromnode> <fromport>nbbranches</fromport>
|
|
88
|
+
<tonode>OptimizerLoop0</tonode> <toport>nbBranches</toport>
|
|
89
|
+
</datalink>
|
|
90
|
+
<datalink control="false">
|
|
91
|
+
<fromnode>Initialisation</fromnode> <fromport>studymodule</fromport>
|
|
92
|
+
<tonode>OptimizerLoop0.Solver</tonode> <toport>studymodule</toport>
|
|
93
|
+
</datalink>
|
|
94
|
+
<datalink control="false">
|
|
95
|
+
<fromnode>OptimizerLoop0.Solver</fromnode> <fromport>o0</fromport>
|
|
96
|
+
<tonode>OptimizerLoop0</tonode> <toport>evalResults</toport>
|
|
97
|
+
</datalink>
|
|
98
|
+
<parameter>
|
|
99
|
+
<tonode>OptimizerLoop0</tonode><toport>nbBranches</toport>
|
|
100
|
+
<value><int>4</int></value>
|
|
101
|
+
</parameter>
|
|
102
|
+
<presentation name="OptimizerLoop0" x="180.5" y="34.5" width="170" height="214" expanded="1" expx="180.5" expy="34.5" expWidth="170" expHeight="214" shownState="0"/>
|
|
103
|
+
<presentation name="OptimizerLoop0.Solver" x="8" y="120" width="158" height="90" expanded="1" expx="8" expy="120" expWidth="158" expHeight="90" shownState="0"/>
|
|
104
|
+
<presentation name="Initialisation" x="14" y="34" width="158" height="90" expanded="1" expx="14" expy="34" expWidth="158" expHeight="90" shownState="0"/>
|
|
105
|
+
<presentation name="__ROOT__" x="0" y="0" width="354.5" height="252.5" expanded="1" expx="0" expy="0" expWidth="354.5" expHeight="252.5" shownState="0"/>
|
|
106
|
+
</proc>
|