pydefx 9.14.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- mpmcn.py +154 -0
- pydefx/__init__.py +32 -0
- pydefx/allpurposebuilder.py +56 -0
- pydefx/configuration.py +120 -0
- pydefx/defaultschemabuilder.py +55 -0
- pydefx/localbuilder.py +37 -0
- pydefx/localstudy.py +78 -0
- pydefx/multijobbuilder.py +33 -0
- pydefx/multijobstudy.py +84 -0
- pydefx/parameters.py +169 -0
- pydefx/plugins/jobexecutor.py +130 -0
- pydefx/plugins/lightexecutor.py +40 -0
- pydefx/plugins/localexecutor.py +81 -0
- pydefx/plugins/mainjob.py +56 -0
- pydefx/plugins/pointeval.py +39 -0
- pydefx/plugins/srunexecutor.py +87 -0
- pydefx/pyscript.py +110 -0
- pydefx/pystudy.py +327 -0
- pydefx/salome_proxy.py +88 -0
- pydefx/sample.py +263 -0
- pydefx/samplecsviterator.py +191 -0
- pydefx/samplecsvmanager.py +136 -0
- pydefx/schemas/idefix_pyschema.xml +106 -0
- pydefx/schemas/plugin.py +81 -0
- pydefx/slurmbuilder.py +33 -0
- pydefx/slurmstudy.py +77 -0
- pydefx/studyexception.py +41 -0
- pydefx/studyresult.py +54 -0
- pydefx-9.14.0.dist-info/METADATA +22 -0
- pydefx-9.14.0.dist-info/RECORD +45 -0
- pydefx-9.14.0.dist-info/WHEEL +4 -0
- salome/bin/salome/test/CTestTestfile.cmake +28 -0
- salome/bin/salome/test/cpp/CTestTestfile.cmake +36 -0
- salome/bin/salome/test/cpp/SampleTest +0 -0
- salome/bin/salome/test/cpp/StudyGeneralTest +0 -0
- salome/bin/salome/test/cpp/StudyRestartTest +0 -0
- salome/bin/salome/test/pyexample/CTestTestfile.cmake +25 -0
- salome/bin/salome/test/pyexample/insitu/insituiterator.py +48 -0
- salome/bin/salome/test/pyexample/insitu/insitumanager.py +46 -0
- salome/bin/salome/test/pyexample/runUnitTest.sh +26 -0
- salome/bin/salome/test/pyexample/test_default.py +79 -0
- salome/bin/salome/test/pyexample/test_insitu.py +60 -0
- salome/bin/salome/test/pyexample/test_mpmcn.py +40 -0
- salome/bin/salome/test/pyexample/test_prescript.py +52 -0
- salome/bin/salome/test/pyexample/test_ydefx_base.py +79 -0
mpmcn.py
ADDED
|
@@ -0,0 +1,154 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
# -*- coding: utf-8 -*-
|
|
3
|
+
# Copyright (C) 2022-2024 EDF
|
|
4
|
+
#
|
|
5
|
+
# This library is free software; you can redistribute it and/or
|
|
6
|
+
# modify it under the terms of the GNU Lesser General Public
|
|
7
|
+
# License as published by the Free Software Foundation; either
|
|
8
|
+
# version 2.1 of the License, or (at your option) any later version.
|
|
9
|
+
#
|
|
10
|
+
# This library is distributed in the hope that it will be useful,
|
|
11
|
+
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
12
|
+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
|
13
|
+
# Lesser General Public License for more details.
|
|
14
|
+
#
|
|
15
|
+
# You should have received a copy of the GNU Lesser General Public
|
|
16
|
+
# License along with this library; if not, write to the Free Software
|
|
17
|
+
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
|
18
|
+
#
|
|
19
|
+
# See http://www.salome-platform.org/ or email : webmaster.salome@opencascade.com
|
|
20
|
+
|
|
21
|
+
import pydefx
|
|
22
|
+
|
|
23
|
+
class Pool:
|
|
24
|
+
def __init__(self, params):
|
|
25
|
+
"""
|
|
26
|
+
:param params: parameter of the job
|
|
27
|
+
:type params: pydefx.Parameters
|
|
28
|
+
"""
|
|
29
|
+
self.myStudy = None
|
|
30
|
+
self.myScript = pydefx.PyScript()
|
|
31
|
+
self.mySample = None
|
|
32
|
+
self.myParams = params
|
|
33
|
+
self.removeTmpDir = False
|
|
34
|
+
|
|
35
|
+
def getResultDirectory(self):
|
|
36
|
+
return self.myParams.salome_parameters.result_directory
|
|
37
|
+
|
|
38
|
+
def map(self, func, iterable):
|
|
39
|
+
if len(iterable) == 0:
|
|
40
|
+
return []
|
|
41
|
+
#
|
|
42
|
+
import inspect
|
|
43
|
+
if not inspect.isfunction(func):
|
|
44
|
+
raise RuntimeError("Input is expected to be a function")
|
|
45
|
+
import importlib
|
|
46
|
+
fModule = importlib.import_module(func.__module__)
|
|
47
|
+
if fModule == "__main__":
|
|
48
|
+
raise RuntimeError("Input function is expected to be part of a module")
|
|
49
|
+
st = None
|
|
50
|
+
with open(fModule.__file__,"r") as ff:
|
|
51
|
+
st = ff.read()
|
|
52
|
+
# retrieve the varname holding the function in its module
|
|
53
|
+
fStr = func.__code__.co_name
|
|
54
|
+
if fStr is None:
|
|
55
|
+
raise RuntimeError("Impossible to locate function in the module containing it !")
|
|
56
|
+
# retrieve args of the func passed in input
|
|
57
|
+
fArgs = inspect.getfullargspec(func).args
|
|
58
|
+
# agregate the content of the Python module containing the function with expected _exec function for pydfx
|
|
59
|
+
pyScript = """{}
|
|
60
|
+
def _exec({}):
|
|
61
|
+
yxyx = {}({})
|
|
62
|
+
return yxyx
|
|
63
|
+
""".format(st,", ".join(fArgs),fStr,", ".join(fArgs))
|
|
64
|
+
#
|
|
65
|
+
self.myScript.loadString(pyScript)
|
|
66
|
+
self.mySample = self.myScript.CreateEmptySample()
|
|
67
|
+
# management of the case of single input
|
|
68
|
+
if not hasattr(iterable[0],"__iter__"):
|
|
69
|
+
iterable = [[elt] for elt in iterable]
|
|
70
|
+
#
|
|
71
|
+
self.mySample.setInputValues( {k:v for k,*v in zip(fArgs,*iterable)} )
|
|
72
|
+
#
|
|
73
|
+
self.myStudy.createNewJob(self.myScript, self.mySample, self.myParams)
|
|
74
|
+
#
|
|
75
|
+
self.myStudy.launch()
|
|
76
|
+
self.myStudy.wait()
|
|
77
|
+
# ask for result : this call implicitely copy back results to the client
|
|
78
|
+
self.myStudy.getResult()
|
|
79
|
+
#
|
|
80
|
+
if self.myStudy.getJobState() == "FINISHED" and self.myStudy.global_result.exit_code == "0":
|
|
81
|
+
messageFromSlaves = self.myStudy.sample.getMessages()
|
|
82
|
+
if all( [elt == "" for elt in messageFromSlaves] ):
|
|
83
|
+
ret = [elt for elt in zip(*[self.myStudy.sample.getOutput(n) for n in self.myStudy.sample.getOutputNames()])]
|
|
84
|
+
if len(self.myStudy.sample.getOutputNames()) == 1:
|
|
85
|
+
ret = [elt[0] for elt in ret]
|
|
86
|
+
self.removeTmpDir = True
|
|
87
|
+
return ret
|
|
88
|
+
else:
|
|
89
|
+
excMsg = "\n".join(["Error for sample # {} : \'{}\' ".format(i,elt) for i,elt in enumerate(messageFromSlaves) if elt != ""])
|
|
90
|
+
excMsg += "\nDirectory containing information for debug : {}".format(self.getResultDirectory())
|
|
91
|
+
exc = RuntimeError( excMsg )
|
|
92
|
+
exc.tmp_dir = self.getResultDirectory()
|
|
93
|
+
raise exc
|
|
94
|
+
else:
|
|
95
|
+
raise RuntimeError( "Error during job submission or during the driver execution (that should never happend). Internal error : {}".format(self.myStudy.getResult().getErrors()) )
|
|
96
|
+
|
|
97
|
+
def __enter__(self):
|
|
98
|
+
self.myStudy = pydefx.PyStudy()
|
|
99
|
+
return self
|
|
100
|
+
|
|
101
|
+
def __exit__(self, exc_type, exc_val, exc_tb):
|
|
102
|
+
import os
|
|
103
|
+
from glob import glob
|
|
104
|
+
import shutil
|
|
105
|
+
if self.removeTmpDir:
|
|
106
|
+
for fn in glob(os.path.join(self.getResultDirectory(),"*")):
|
|
107
|
+
if os.path.isdir( fn ):
|
|
108
|
+
shutil.rmtree( fn)
|
|
109
|
+
else:
|
|
110
|
+
os.unlink( fn )
|
|
111
|
+
pass
|
|
112
|
+
pass
|
|
113
|
+
|
|
114
|
+
def init(resourceName, resultDirectory = "/tmp"):
|
|
115
|
+
"""
|
|
116
|
+
Instanciate a pydefx.Parameters intance that can be overriden right after.
|
|
117
|
+
Here some example of typical override of the returned object of this method :
|
|
118
|
+
|
|
119
|
+
define a local directory for working files :
|
|
120
|
+
myParams.salome_parameters.result_directory with existing directory. If not myParams.createResultDirectory("/tmp")
|
|
121
|
+
|
|
122
|
+
Define additionnal files necessary for computation.
|
|
123
|
+
They will be copied to the remote working directory.
|
|
124
|
+
This parameter is empty by default :
|
|
125
|
+
myParams.salome_parameters.in_files = []
|
|
126
|
+
|
|
127
|
+
Override computation ressource :
|
|
128
|
+
myParams.salome_parameters.resource_required.name)
|
|
129
|
+
|
|
130
|
+
Override default # of parallel evaluation :
|
|
131
|
+
myParams.nb_branches
|
|
132
|
+
|
|
133
|
+
Override number of cores requested by the job when job resource is a cluster (for job scheduler query) :
|
|
134
|
+
myParams.salome_parameters.resource_required.nb_proc
|
|
135
|
+
|
|
136
|
+
Override number of computationnal nodes on cluster to be allocated when job resource is a cluster (for job scheduler query) :
|
|
137
|
+
myParams.salome_parameters.resource_required.nb_node
|
|
138
|
+
|
|
139
|
+
Override working directory :
|
|
140
|
+
myParams.salome_parameters.work_directory
|
|
141
|
+
|
|
142
|
+
:param resourceName: Name of the resource matching one of the ${KERNEL_ROOT_DIR}/share/salome/resources/kernel/CatalogResources.xml
|
|
143
|
+
:param resultDirectory: Directory used to transfer results
|
|
144
|
+
:return: a pydefx.Parameters instance
|
|
145
|
+
"""
|
|
146
|
+
myParams = pydefx.Parameters()
|
|
147
|
+
from pydefx import configuration
|
|
148
|
+
if resourceName not in configuration.availableResources():
|
|
149
|
+
raise RuntimeError("Resource \"{}\" is not existing or not declared as able to launch job. Available resources are : {}".format(resourceName,str(configuration.availableResources())))
|
|
150
|
+
myParams.configureResource(resourceName)
|
|
151
|
+
myParams.createResultDirectory(resultDirectory)
|
|
152
|
+
if resourceName == "localhost":
|
|
153
|
+
myParams.nb_branches = configuration.allCoresAvailable()
|
|
154
|
+
return myParams
|
pydefx/__init__.py
ADDED
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
# Copyright (C) 2019-2024 EDF
|
|
2
|
+
#
|
|
3
|
+
# This library is free software; you can redistribute it and/or
|
|
4
|
+
# modify it under the terms of the GNU Lesser General Public
|
|
5
|
+
# License as published by the Free Software Foundation; either
|
|
6
|
+
# version 2.1 of the License, or (at your option) any later version.
|
|
7
|
+
#
|
|
8
|
+
# This library is distributed in the hope that it will be useful,
|
|
9
|
+
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
10
|
+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
|
11
|
+
# Lesser General Public License for more details.
|
|
12
|
+
#
|
|
13
|
+
# You should have received a copy of the GNU Lesser General Public
|
|
14
|
+
# License along with this library; if not, write to the Free Software
|
|
15
|
+
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
|
16
|
+
#
|
|
17
|
+
# See http://www.salome-platform.org/ or email : webmaster.salome@opencascade.com
|
|
18
|
+
|
|
19
|
+
from .parameters import Parameters
|
|
20
|
+
from .pyscript import PyScript
|
|
21
|
+
from .pystudy import PyStudy
|
|
22
|
+
from .sample import Sample
|
|
23
|
+
from .defaultschemabuilder import DefaultSchemaBuilder
|
|
24
|
+
from .allpurposebuilder import AllPurposeBuilder
|
|
25
|
+
from .localbuilder import LocalBuilder
|
|
26
|
+
from .multijobbuilder import MultiJobBuilder
|
|
27
|
+
from .slurmbuilder import SlurmBuilder
|
|
28
|
+
|
|
29
|
+
from .salome_proxy import forceSalomeServers, forceNoSalomeServers
|
|
30
|
+
from .multijobstudy import MultiJobStudy
|
|
31
|
+
from .slurmstudy import SlurmStudy
|
|
32
|
+
from .localstudy import LocalStudy
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
# -*- coding: utf-8 -*-
|
|
2
|
+
# Copyright (C) 2019-2024 EDF
|
|
3
|
+
#
|
|
4
|
+
# This library is free software; you can redistribute it and/or
|
|
5
|
+
# modify it under the terms of the GNU Lesser General Public
|
|
6
|
+
# License as published by the Free Software Foundation; either
|
|
7
|
+
# version 2.1 of the License, or (at your option) any later version.
|
|
8
|
+
#
|
|
9
|
+
# This library is distributed in the hope that it will be useful,
|
|
10
|
+
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
11
|
+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
|
12
|
+
# Lesser General Public License for more details.
|
|
13
|
+
#
|
|
14
|
+
# You should have received a copy of the GNU Lesser General Public
|
|
15
|
+
# License along with this library; if not, write to the Free Software
|
|
16
|
+
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
|
17
|
+
#
|
|
18
|
+
# See http://www.salome-platform.org/ or email : webmaster.salome@opencascade.com
|
|
19
|
+
#
|
|
20
|
+
import inspect
|
|
21
|
+
import pathlib
|
|
22
|
+
import os
|
|
23
|
+
|
|
24
|
+
class AllPurposeBuilder:
|
|
25
|
+
def __init__(self, executor = None, pointEval = None, mainJob = None):
|
|
26
|
+
filename = inspect.getframeinfo(inspect.currentframe()).filename
|
|
27
|
+
install_root_directory = pathlib.Path(filename).resolve().parent
|
|
28
|
+
install_files_directory = os.path.join(install_root_directory, "plugins")
|
|
29
|
+
|
|
30
|
+
if executor is None:
|
|
31
|
+
raise TypeError("Parameter executor should not be None.")
|
|
32
|
+
self.executor = executor
|
|
33
|
+
|
|
34
|
+
if pointEval is None:
|
|
35
|
+
pointEval = os.path.join(install_files_directory, "pointeval.py")
|
|
36
|
+
self.pointEval = pointEval
|
|
37
|
+
|
|
38
|
+
if mainJob is None:
|
|
39
|
+
mainJob = os.path.join(install_files_directory, "mainjob.py")
|
|
40
|
+
self.mainJob = mainJob
|
|
41
|
+
|
|
42
|
+
def getMainJob(self):
|
|
43
|
+
return self.mainJob
|
|
44
|
+
|
|
45
|
+
def getExecutor(self):
|
|
46
|
+
return self.executor
|
|
47
|
+
|
|
48
|
+
def getPointEval(self):
|
|
49
|
+
return self.pointEval
|
|
50
|
+
|
|
51
|
+
def getPluginName(self):
|
|
52
|
+
basename = os.path.basename(self.executor)
|
|
53
|
+
if not basename.endswith(".py"):
|
|
54
|
+
raise ValueError("File name {} does not end with '.py'.".format(
|
|
55
|
+
self.executor))
|
|
56
|
+
return basename[:-3]
|
pydefx/configuration.py
ADDED
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
# Copyright (C) 2019-2024 EDF
|
|
2
|
+
#
|
|
3
|
+
# This library is free software; you can redistribute it and/or
|
|
4
|
+
# modify it under the terms of the GNU Lesser General Public
|
|
5
|
+
# License as published by the Free Software Foundation; either
|
|
6
|
+
# version 2.1 of the License, or (at your option) any later version.
|
|
7
|
+
#
|
|
8
|
+
# This library is distributed in the hope that it will be useful,
|
|
9
|
+
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
10
|
+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
|
11
|
+
# Lesser General Public License for more details.
|
|
12
|
+
#
|
|
13
|
+
# You should have received a copy of the GNU Lesser General Public
|
|
14
|
+
# License along with this library; if not, write to the Free Software
|
|
15
|
+
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
|
16
|
+
#
|
|
17
|
+
# See http://www.salome-platform.org/ or email : webmaster.salome@opencascade.com
|
|
18
|
+
#
|
|
19
|
+
from . import salome_proxy
|
|
20
|
+
from . import parameters
|
|
21
|
+
import tempfile
|
|
22
|
+
import pathlib
|
|
23
|
+
import os
|
|
24
|
+
import json
|
|
25
|
+
|
|
26
|
+
def defaultWorkingDir(resource):
|
|
27
|
+
resManager = salome_proxy.getResourcesManager()
|
|
28
|
+
resource_definition = resManager.GetResourceDefinition(resource)
|
|
29
|
+
return resource_definition.working_directory
|
|
30
|
+
|
|
31
|
+
def defaultNbBranches(resource):
|
|
32
|
+
"""
|
|
33
|
+
Return the number of cores available on a resource.
|
|
34
|
+
"""
|
|
35
|
+
resManager = salome_proxy.getResourcesManager()
|
|
36
|
+
resource_definition = resManager.GetResourceDefinition(resource)
|
|
37
|
+
ret = resource_definition.nb_node * resource_definition.nb_proc_per_node
|
|
38
|
+
if ret < 1:
|
|
39
|
+
ret = 1
|
|
40
|
+
return ret
|
|
41
|
+
|
|
42
|
+
def allCoresAvailable():
|
|
43
|
+
"""
|
|
44
|
+
Return the total number of cores of all resources that can run containers.
|
|
45
|
+
( "canRunContainers" attribute set to true in CatalogResources.xml )
|
|
46
|
+
"""
|
|
47
|
+
resManager = salome_proxy.getResourcesManager()
|
|
48
|
+
params = salome_proxy.createSalomeParameters()
|
|
49
|
+
params.resource_required.can_run_containers = True
|
|
50
|
+
resources = resManager.GetFittingResources(params.resource_required)
|
|
51
|
+
return sum([defaultNbBranches(res) for res in resources ])
|
|
52
|
+
|
|
53
|
+
def defaultBaseDirectory():
|
|
54
|
+
"""Return the default path to the root of any new result directory."""
|
|
55
|
+
return str(pathlib.Path.home())
|
|
56
|
+
|
|
57
|
+
def newResultDirectory(basedir=None):
|
|
58
|
+
""" A new directory is created and the path is returned."""
|
|
59
|
+
if basedir is None :
|
|
60
|
+
basedir = defaultBaseDirectory()
|
|
61
|
+
return tempfile.mkdtemp(prefix='idefix',dir=basedir)
|
|
62
|
+
|
|
63
|
+
def defaultWckey(resource="localhost"):
|
|
64
|
+
result = ""
|
|
65
|
+
if resource != "localhost":
|
|
66
|
+
result = "P120K:SALOME"
|
|
67
|
+
return result
|
|
68
|
+
|
|
69
|
+
def availableResources():
|
|
70
|
+
"""
|
|
71
|
+
Return the list of resources defined in the current catalog that are able to
|
|
72
|
+
launch jobs.
|
|
73
|
+
Ydefx can launch the evaluations in a job on one of these resources.
|
|
74
|
+
"""
|
|
75
|
+
resManager = salome_proxy.getResourcesManager()
|
|
76
|
+
params = salome_proxy.createSalomeParameters()
|
|
77
|
+
params.resource_required.can_launch_batch_jobs = True
|
|
78
|
+
# GetFittingResources returns a tuple if in no salome session mode.
|
|
79
|
+
# Force to list for uniformity between the two modes.
|
|
80
|
+
return list(resManager.GetFittingResources(params.resource_required))
|
|
81
|
+
|
|
82
|
+
def exportConfig(dicconfig, directory = None):
|
|
83
|
+
""" Save the configuration to a directory.
|
|
84
|
+
dicconfig is a dictionary which contains the parameters to be saved.
|
|
85
|
+
If directory is None, the configuration is saved to the current directory.
|
|
86
|
+
Return the path to the configuration file.
|
|
87
|
+
"""
|
|
88
|
+
if directory is None:
|
|
89
|
+
directory = os.getcwd()
|
|
90
|
+
configpath = os.path.join(directory, "idefixconfig.json")
|
|
91
|
+
with open(configpath, "w") as f:
|
|
92
|
+
json.dump(dicconfig, f, indent=2)
|
|
93
|
+
return configpath
|
|
94
|
+
|
|
95
|
+
def loadConfig(directory = None):
|
|
96
|
+
""" Return the configuration dictionary from a directory.
|
|
97
|
+
If the directory is None, use the current directory.
|
|
98
|
+
"""
|
|
99
|
+
if directory is None:
|
|
100
|
+
directory = os.getcwd()
|
|
101
|
+
configpath = os.path.join(directory, "idefixconfig.json")
|
|
102
|
+
if not pathlib.Path(configpath).is_file():
|
|
103
|
+
configpath = os.path.join(directory, "..", "idefixconfig.json")
|
|
104
|
+
if not pathlib.Path(configpath).is_file():
|
|
105
|
+
message = "Configuration file not found in directory " + str(directory)
|
|
106
|
+
raise FileNotFoundError(message)
|
|
107
|
+
with open(configpath, "r") as f:
|
|
108
|
+
config = json.load(f)
|
|
109
|
+
return config
|
|
110
|
+
|
|
111
|
+
def loadJobConfig(directory = None):
|
|
112
|
+
""" Return the salome job parameters loaded from a directory which contains
|
|
113
|
+
a idefixconfig.json file.
|
|
114
|
+
If the directory is None, use the current directory.
|
|
115
|
+
"""
|
|
116
|
+
config = loadConfig(directory)
|
|
117
|
+
params = parameters.Parameters()
|
|
118
|
+
params.loadDict(config["params"])
|
|
119
|
+
result = params.salome_parameters
|
|
120
|
+
return result
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
# -*- coding: utf-8 -*-
|
|
2
|
+
# Copyright (C) 2019-2024 EDF
|
|
3
|
+
#
|
|
4
|
+
# This library is free software; you can redistribute it and/or
|
|
5
|
+
# modify it under the terms of the GNU Lesser General Public
|
|
6
|
+
# License as published by the Free Software Foundation; either
|
|
7
|
+
# version 2.1 of the License, or (at your option) any later version.
|
|
8
|
+
#
|
|
9
|
+
# This library is distributed in the hope that it will be useful,
|
|
10
|
+
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
11
|
+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
|
12
|
+
# Lesser General Public License for more details.
|
|
13
|
+
#
|
|
14
|
+
# You should have received a copy of the GNU Lesser General Public
|
|
15
|
+
# License along with this library; if not, write to the Free Software
|
|
16
|
+
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
|
17
|
+
#
|
|
18
|
+
# See http://www.salome-platform.org/ or email : webmaster.salome@opencascade.com
|
|
19
|
+
#
|
|
20
|
+
import inspect
|
|
21
|
+
import pathlib
|
|
22
|
+
import os
|
|
23
|
+
import shutil
|
|
24
|
+
|
|
25
|
+
class DefaultSchemaBuilder:
|
|
26
|
+
def __init__(self, prescript=None):
|
|
27
|
+
"""
|
|
28
|
+
This object builds the YACS schema for the parametric computation.
|
|
29
|
+
prescript: contains python code that is executed before any evaluation.
|
|
30
|
+
"""
|
|
31
|
+
self.prescript = prescript
|
|
32
|
+
|
|
33
|
+
def buildSchema(self, local_work_dir):
|
|
34
|
+
"""
|
|
35
|
+
Create the YACS schema and copy it to local_work_dir.
|
|
36
|
+
local_work_dir : path where the schema will be created.
|
|
37
|
+
return:
|
|
38
|
+
path to the created file,
|
|
39
|
+
list of additional files needed for running.
|
|
40
|
+
"""
|
|
41
|
+
# use generic schema
|
|
42
|
+
filename = inspect.getframeinfo(inspect.currentframe()).filename
|
|
43
|
+
install_directory = pathlib.Path(filename).resolve().parent
|
|
44
|
+
yacs_schema_path = os.path.join(install_directory, "schemas",
|
|
45
|
+
"idefix_pyschema.xml")
|
|
46
|
+
plugin_path = os.path.join(install_directory, "schemas", "plugin.py")
|
|
47
|
+
yacs_schema_path = shutil.copy(yacs_schema_path, local_work_dir)
|
|
48
|
+
plugin_path = shutil.copy(plugin_path, local_work_dir)
|
|
49
|
+
files = [plugin_path]
|
|
50
|
+
if self.prescript:
|
|
51
|
+
prescript_path = os.path.join(local_work_dir, "idefix_prescript.py")
|
|
52
|
+
with open(prescript_path, "w") as f:
|
|
53
|
+
f.write(self.prescript)
|
|
54
|
+
files.append(prescript_path)
|
|
55
|
+
return yacs_schema_path, files
|
pydefx/localbuilder.py
ADDED
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
# -*- coding: utf-8 -*-
|
|
2
|
+
# Copyright (C) 2019-2024 EDF
|
|
3
|
+
#
|
|
4
|
+
# This library is free software; you can redistribute it and/or
|
|
5
|
+
# modify it under the terms of the GNU Lesser General Public
|
|
6
|
+
# License as published by the Free Software Foundation; either
|
|
7
|
+
# version 2.1 of the License, or (at your option) any later version.
|
|
8
|
+
#
|
|
9
|
+
# This library is distributed in the hope that it will be useful,
|
|
10
|
+
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
11
|
+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
|
12
|
+
# Lesser General Public License for more details.
|
|
13
|
+
#
|
|
14
|
+
# You should have received a copy of the GNU Lesser General Public
|
|
15
|
+
# License along with this library; if not, write to the Free Software
|
|
16
|
+
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
|
17
|
+
#
|
|
18
|
+
# See http://www.salome-platform.org/ or email : webmaster.salome@opencascade.com
|
|
19
|
+
#
|
|
20
|
+
import inspect
|
|
21
|
+
import pathlib
|
|
22
|
+
import os
|
|
23
|
+
from .allpurposebuilder import AllPurposeBuilder
|
|
24
|
+
|
|
25
|
+
class LocalBuilder(AllPurposeBuilder):
|
|
26
|
+
def __init__(self, executor = None, pointEval = None, mainJob = None):
|
|
27
|
+
filename = inspect.getframeinfo(inspect.currentframe()).filename
|
|
28
|
+
install_root_directory = pathlib.Path(filename).resolve().parent
|
|
29
|
+
install_files_directory = os.path.join(install_root_directory, "plugins")
|
|
30
|
+
if executor is None:
|
|
31
|
+
executor = os.path.join(install_files_directory, "localexecutor.py")
|
|
32
|
+
elif executor == "localexecutor" or executor == "localexecutor.py":
|
|
33
|
+
executor = os.path.join(install_files_directory, "localexecutor.py")
|
|
34
|
+
elif executor == "lightexecutor" or executor == "lightexecutor.py":
|
|
35
|
+
executor = os.path.join(install_files_directory, "lightexecutor.py")
|
|
36
|
+
super().__init__(executor, pointEval, mainJob)
|
|
37
|
+
|
pydefx/localstudy.py
ADDED
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
# -*- coding: utf-8 -*-
|
|
2
|
+
# Copyright (C) 2019-2024 EDF
|
|
3
|
+
#
|
|
4
|
+
# This library is free software; you can redistribute it and/or
|
|
5
|
+
# modify it under the terms of the GNU Lesser General Public
|
|
6
|
+
# License as published by the Free Software Foundation; either
|
|
7
|
+
# version 2.1 of the License, or (at your option) any later version.
|
|
8
|
+
#
|
|
9
|
+
# This library is distributed in the hope that it will be useful,
|
|
10
|
+
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
11
|
+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
|
12
|
+
# Lesser General Public License for more details.
|
|
13
|
+
#
|
|
14
|
+
# You should have received a copy of the GNU Lesser General Public
|
|
15
|
+
# License along with this library; if not, write to the Free Software
|
|
16
|
+
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
|
17
|
+
#
|
|
18
|
+
# See http://www.salome-platform.org/ or email : webmaster.salome@opencascade.com
|
|
19
|
+
#
|
|
20
|
+
import copy
|
|
21
|
+
import os
|
|
22
|
+
import json
|
|
23
|
+
from . import pystudy
|
|
24
|
+
from . import localbuilder
|
|
25
|
+
from . import salome_proxy
|
|
26
|
+
from . import configuration
|
|
27
|
+
|
|
28
|
+
class LocalStudy(pystudy.PyStudy):
|
|
29
|
+
"""
|
|
30
|
+
This study is always localy evaluated.
|
|
31
|
+
"""
|
|
32
|
+
def __init__(self, sampleManager=None, schemaBuilder=None):
|
|
33
|
+
if schemaBuilder is None:
|
|
34
|
+
schemaBuilder = localbuilder.LocalBuilder()
|
|
35
|
+
super().__init__(sampleManager, schemaBuilder)
|
|
36
|
+
|
|
37
|
+
def createNewJob(self, script, sample, params):
|
|
38
|
+
self._check(script,sample)
|
|
39
|
+
self.sample = sample
|
|
40
|
+
self.params = copy.deepcopy(params)
|
|
41
|
+
# dump the remote jobs parameters to the configuration file
|
|
42
|
+
params_dic = params.dumpDict()
|
|
43
|
+
# modify the parameters for the local loop job
|
|
44
|
+
self.params.salome_parameters.resource_required.name = "localhost"
|
|
45
|
+
self.params.salome_parameters.job_type = "command_salome" #"python_salome"
|
|
46
|
+
self.params.createTmpResultDirectory()
|
|
47
|
+
result_directory = self.params.salome_parameters.result_directory
|
|
48
|
+
# export sample to result_directory
|
|
49
|
+
inputFiles = self.sampleManager.prepareRun(self.sample, result_directory)
|
|
50
|
+
inputFiles.extend([self.schemaBuilder.getExecutor(),
|
|
51
|
+
self.schemaBuilder.getPointEval()])
|
|
52
|
+
self.params.salome_parameters.job_file = self.schemaBuilder.getMainJob()
|
|
53
|
+
|
|
54
|
+
# export config
|
|
55
|
+
dicconfig = {}
|
|
56
|
+
dicconfig["nbbranches"] = self.params.nb_branches
|
|
57
|
+
dicconfig["studymodule"] = "idefixstudy"
|
|
58
|
+
dicconfig["sampleIterator"] = self.sampleManager.getModuleName()
|
|
59
|
+
dicconfig["params"] = params_dic
|
|
60
|
+
dicconfig["plugin"] = self.schemaBuilder.getPluginName()
|
|
61
|
+
configpath = configuration.exportConfig(dicconfig, result_directory)
|
|
62
|
+
studypath = os.path.join(result_directory, "idefixstudy.py")
|
|
63
|
+
with open(studypath, "w") as f:
|
|
64
|
+
f.write(script.script)
|
|
65
|
+
|
|
66
|
+
inputFiles.extend([configpath, studypath])
|
|
67
|
+
|
|
68
|
+
# this list manipulation is needed because in_files is not a python list
|
|
69
|
+
# if we don't use a salome session. In that case swig uses a python tuple
|
|
70
|
+
# in order to map a std::list as a parameter of a structure.
|
|
71
|
+
in_files_as_list = list(self.params.salome_parameters.in_files)
|
|
72
|
+
self.params.salome_parameters.in_files = in_files_as_list + inputFiles
|
|
73
|
+
launcher = salome_proxy.getLauncher()
|
|
74
|
+
self.job_id = launcher.createJob(self.params.salome_parameters)
|
|
75
|
+
return self.job_id
|
|
76
|
+
|
|
77
|
+
def jobType(self):
|
|
78
|
+
return "command_salome"
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
# -*- coding: utf-8 -*-
|
|
2
|
+
# Copyright (C) 2019-2024 EDF
|
|
3
|
+
#
|
|
4
|
+
# This library is free software; you can redistribute it and/or
|
|
5
|
+
# modify it under the terms of the GNU Lesser General Public
|
|
6
|
+
# License as published by the Free Software Foundation; either
|
|
7
|
+
# version 2.1 of the License, or (at your option) any later version.
|
|
8
|
+
#
|
|
9
|
+
# This library is distributed in the hope that it will be useful,
|
|
10
|
+
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
11
|
+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
|
12
|
+
# Lesser General Public License for more details.
|
|
13
|
+
#
|
|
14
|
+
# You should have received a copy of the GNU Lesser General Public
|
|
15
|
+
# License along with this library; if not, write to the Free Software
|
|
16
|
+
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
|
17
|
+
#
|
|
18
|
+
# See http://www.salome-platform.org/ or email : webmaster.salome@opencascade.com
|
|
19
|
+
#
|
|
20
|
+
import inspect
|
|
21
|
+
import pathlib
|
|
22
|
+
import os
|
|
23
|
+
from .allpurposebuilder import AllPurposeBuilder
|
|
24
|
+
|
|
25
|
+
class MultiJobBuilder(AllPurposeBuilder):
|
|
26
|
+
def __init__(self, executor = None, pointEval = None, mainJob = None):
|
|
27
|
+
filename = inspect.getframeinfo(inspect.currentframe()).filename
|
|
28
|
+
install_root_directory = pathlib.Path(filename).resolve().parent
|
|
29
|
+
install_files_directory = os.path.join(install_root_directory, "plugins")
|
|
30
|
+
|
|
31
|
+
if executor is None:
|
|
32
|
+
executor = os.path.join(install_files_directory, "jobexecutor.py")
|
|
33
|
+
super().__init__(executor, pointEval, mainJob)
|
pydefx/multijobstudy.py
ADDED
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
# -*- coding: utf-8 -*-
|
|
2
|
+
# Copyright (C) 2019-2024 EDF
|
|
3
|
+
#
|
|
4
|
+
# This library is free software; you can redistribute it and/or
|
|
5
|
+
# modify it under the terms of the GNU Lesser General Public
|
|
6
|
+
# License as published by the Free Software Foundation; either
|
|
7
|
+
# version 2.1 of the License, or (at your option) any later version.
|
|
8
|
+
#
|
|
9
|
+
# This library is distributed in the hope that it will be useful,
|
|
10
|
+
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
11
|
+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
|
12
|
+
# Lesser General Public License for more details.
|
|
13
|
+
#
|
|
14
|
+
# You should have received a copy of the GNU Lesser General Public
|
|
15
|
+
# License along with this library; if not, write to the Free Software
|
|
16
|
+
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
|
17
|
+
#
|
|
18
|
+
# See http://www.salome-platform.org/ or email : webmaster.salome@opencascade.com
|
|
19
|
+
#
|
|
20
|
+
import copy
|
|
21
|
+
import os
|
|
22
|
+
import json
|
|
23
|
+
from . import pystudy
|
|
24
|
+
from . import multijobbuilder
|
|
25
|
+
from . import salome_proxy
|
|
26
|
+
from . import configuration
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
class MultiJobStudy(pystudy.PyStudy):
|
|
30
|
+
"""
|
|
31
|
+
This study uses one different job for each evaluation.
|
|
32
|
+
"""
|
|
33
|
+
def __init__(self, sampleManager=None, schemaBuilder=None):
|
|
34
|
+
if schemaBuilder is None:
|
|
35
|
+
schemaBuilder = multijobbuilder.MultiJobBuilder()
|
|
36
|
+
super().__init__(sampleManager, schemaBuilder)
|
|
37
|
+
|
|
38
|
+
def createNewJob(self, script, sample, params):
|
|
39
|
+
self._check(script,sample)
|
|
40
|
+
self.sample = sample
|
|
41
|
+
self.params = copy.deepcopy(params)
|
|
42
|
+
main_job_work_dir = self.params.salome_parameters.result_directory
|
|
43
|
+
params_dic = params.dumpDict()
|
|
44
|
+
params_dic["salome_parameters"]["job_type"] = "command_salome"
|
|
45
|
+
params_dic["salome_parameters"]["job_file"] = self.schemaBuilder.getPointEval()
|
|
46
|
+
params_dic["salome_parameters"]["local_directory"] = main_job_work_dir
|
|
47
|
+
# set the parameters of the local job
|
|
48
|
+
self.params.salome_parameters.resource_required.name = "localhost"
|
|
49
|
+
self.params.salome_parameters.job_type = "command_salome" #"python_salome"
|
|
50
|
+
|
|
51
|
+
self.params.salome_parameters.work_directory = main_job_work_dir
|
|
52
|
+
self.params.createTmpResultDirectory()
|
|
53
|
+
result_directory = self.params.salome_parameters.result_directory
|
|
54
|
+
# export sample to result_directory
|
|
55
|
+
inputFiles = self.sampleManager.prepareRun(self.sample, result_directory)
|
|
56
|
+
inputFiles.extend([self.schemaBuilder.getExecutor(),
|
|
57
|
+
self.schemaBuilder.getPointEval()])
|
|
58
|
+
self.params.salome_parameters.job_file = self.schemaBuilder.getMainJob()
|
|
59
|
+
|
|
60
|
+
# export config
|
|
61
|
+
dicconfig = {}
|
|
62
|
+
dicconfig["nbbranches"] = self.params.nb_branches
|
|
63
|
+
dicconfig["studymodule"] = "idefixstudy"
|
|
64
|
+
dicconfig["sampleIterator"] = self.sampleManager.getModuleName()
|
|
65
|
+
dicconfig["params"] = params_dic
|
|
66
|
+
dicconfig["plugin"] = self.schemaBuilder.getPluginName()
|
|
67
|
+
configpath = configuration.exportConfig(dicconfig, result_directory)
|
|
68
|
+
studypath = os.path.join(result_directory, "idefixstudy.py")
|
|
69
|
+
with open(studypath, "w") as f:
|
|
70
|
+
f.write(script.script)
|
|
71
|
+
|
|
72
|
+
inputFiles.extend([configpath, studypath])
|
|
73
|
+
|
|
74
|
+
# this list manipulation is needed because in_files is not a python list
|
|
75
|
+
# if we don't use a salome session. In that case swig uses a python tuple
|
|
76
|
+
# in order to map a std::list as a parameter of a structure.
|
|
77
|
+
in_files_as_list = list(self.params.salome_parameters.in_files)
|
|
78
|
+
self.params.salome_parameters.in_files = in_files_as_list + inputFiles
|
|
79
|
+
launcher = salome_proxy.getLauncher()
|
|
80
|
+
self.job_id = launcher.createJob(self.params.salome_parameters)
|
|
81
|
+
return self.job_id
|
|
82
|
+
|
|
83
|
+
def jobType(self):
|
|
84
|
+
return "command_salome"
|