pydefx 9.14.0__py3-none-any.whl

pydefx/parameters.py

@@ -0,0 +1,169 @@
+# -*- coding: utf-8 -*-
+# Copyright (C) 2019-2024 EDF
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307 USA
+#
+# See http://www.salome-platform.org/ or email : webmaster.salome@opencascade.com
+#
+import tempfile
+from . import salome_proxy
+from . import configuration
+
+class Parameters:
+  def __init__(self, resource="localhost",
+               nb_branches=None, salome_parameters=None):
+    if salome_parameters is None :
+      job_params = salome_proxy.createSalomeParameters()
+      job_params.job_type = "yacs_file"
+      job_params.resource_required.name = resource
+      job_params.job_name = "idefix_job"
+      job_params.wckey = configuration.defaultWckey(resource)
+      job_params.work_directory = configuration.defaultWorkingDir(resource)
+      if nb_branches is None:
+        nb_branches = configuration.defaultNbBranches(resource)
+      job_params.resource_required.nb_proc = nb_branches
+      self.nb_branches = nb_branches
+      self.salome_parameters = job_params
+    else:
+      if nb_branches is None:
+        nb_branches = salome_parameters.resource_required.nb_proc
+      self.nb_branches = nb_branches
+      self.salome_parameters = salome_parameters
+
+  def configureResource(self, resource):
+    self.salome_parameters.resource_required.name = resource
+    self.salome_parameters.work_directory = configuration.defaultWorkingDir(
+                                                                       resource)
+    nb_branches = configuration.defaultNbBranches(resource)
+    self.salome_parameters.resource_required.nb_proc = nb_branches
+    self.nb_branches = nb_branches
+    self.salome_parameters.wckey = configuration.defaultWckey(resource)
+
+  def createResultDirectory(self, result_base_dir):
+    self.salome_parameters.result_directory = configuration.newResultDirectory(
+                                                                result_base_dir)
+
+  def createTmpResultDirectory(self):
+    self.salome_parameters.result_directory = configuration.newResultDirectory(
+                                                          tempfile.gettempdir())
+
+  # Specific deep copy function is needed because the default one does not work
+  # for swig objects, when we are in no salome session mode.
+  def __deepcopy__(self, memo):
+    cls = self.__class__
+    newobj = cls.__new__(cls)
+    newobj.nb_branches = self.nb_branches
+    newobj.salome_parameters = salome_proxy.createSalomeParameters()
+    newobj.salome_parameters.job_name = self.salome_parameters.job_name
+    newobj.salome_parameters.job_type = self.salome_parameters.job_type
+    newobj.salome_parameters.job_file = self.salome_parameters.job_file
+    newobj.salome_parameters.pre_command = self.salome_parameters.pre_command
+    newobj.salome_parameters.env_file = self.salome_parameters.env_file
+    newobj.salome_parameters.in_files = list(self.salome_parameters.in_files)
+    newobj.salome_parameters.out_files = list(self.salome_parameters.out_files)
+    newobj.salome_parameters.work_directory = self.salome_parameters.work_directory
+    newobj.salome_parameters.local_directory = self.salome_parameters.local_directory
+    newobj.salome_parameters.result_directory = self.salome_parameters.result_directory
+    newobj.salome_parameters.maximum_duration = self.salome_parameters.maximum_duration
+    newobj.salome_parameters.queue = self.salome_parameters.queue
+    newobj.salome_parameters.partition = self.salome_parameters.partition
+    newobj.salome_parameters.exclusive = self.salome_parameters.exclusive
+    newobj.salome_parameters.mem_per_cpu = self.salome_parameters.mem_per_cpu
+    newobj.salome_parameters.wckey = self.salome_parameters.wckey
+    newobj.salome_parameters.extra_params = self.salome_parameters.extra_params
+    #newobj.salome_parameters.specific_parameters = self.salome_parameters.specific_parameters
+    newobj.salome_parameters.resource_required.name = self.salome_parameters.resource_required.name
+    newobj.salome_parameters.resource_required.hostname = self.salome_parameters.resource_required.hostname
+    newobj.salome_parameters.resource_required.can_launch_batch_jobs = self.salome_parameters.resource_required.can_launch_batch_jobs
+    newobj.salome_parameters.resource_required.can_run_containers = self.salome_parameters.resource_required.can_run_containers
+    newobj.salome_parameters.resource_required.OS = self.salome_parameters.resource_required.OS
+    newobj.salome_parameters.resource_required.nb_proc = self.salome_parameters.resource_required.nb_proc
+    newobj.salome_parameters.resource_required.mem_mb = self.salome_parameters.resource_required.mem_mb
+    newobj.salome_parameters.resource_required.cpu_clock = self.salome_parameters.resource_required.cpu_clock
+    newobj.salome_parameters.resource_required.nb_node = self.salome_parameters.resource_required.nb_node
+    newobj.salome_parameters.resource_required.nb_proc_per_node = self.salome_parameters.resource_required.nb_proc_per_node
+
+    return newobj
+
+  def dumpDict(self):
+    """Create a dictionary with all the properties.
+       Can be used for serialization with json."""
+    newdict = {
+      "nb_branches" : self.nb_branches,
+      "salome_parameters" : {
+          "job_name" : self.salome_parameters.job_name,
+          "job_type" : self.salome_parameters.job_type,
+          "job_file" : self.salome_parameters.job_file,
+          "pre_command" : self.salome_parameters.pre_command,
+          "env_file" : self.salome_parameters.env_file,
+          "in_files" : list(self.salome_parameters.in_files),
+          "out_files" : list(self.salome_parameters.out_files),
+          "work_directory" : self.salome_parameters.work_directory,
+          "local_directory" : self.salome_parameters.local_directory,
+          "result_directory" : self.salome_parameters.result_directory,
+          "maximum_duration" : self.salome_parameters.maximum_duration,
+          "queue" : self.salome_parameters.queue,
+          "partition" : self.salome_parameters.partition,
+          "exclusive" : self.salome_parameters.exclusive,
+          "mem_per_cpu" : self.salome_parameters.mem_per_cpu,
+          "wckey" : self.salome_parameters.wckey,
+          "extra_params" : self.salome_parameters.extra_params,
+          #"specific_parameters" : str(self.salome_parameters.specific_parameters),
+          "resource_required" : {
+              "name" : self.salome_parameters.resource_required.name,
+              "hostname" : self.salome_parameters.resource_required.hostname,
+              "can_launch_batch_jobs" : self.salome_parameters.resource_required.can_launch_batch_jobs,
+              "can_run_containers" : self.salome_parameters.resource_required.can_run_containers,
+              "OS" : self.salome_parameters.resource_required.OS,
+              "nb_proc" : self.salome_parameters.resource_required.nb_proc,
+              "mem_mb" : self.salome_parameters.resource_required.mem_mb,
+              "cpu_clock" : self.salome_parameters.resource_required.cpu_clock,
+              "nb_node" : self.salome_parameters.resource_required.nb_node,
+              "nb_proc_per_node" : self.salome_parameters.resource_required.nb_proc_per_node
+          }
+      }
+    }
+    return newdict
+
+  def loadDict(self, dico):
+    self.nb_branches = dico["nb_branches"]
+    #self.salome_parameters = salome_proxy.createSalomeParameters()
+    self.salome_parameters.job_name = dico["salome_parameters"]["job_name"]
+    self.salome_parameters.job_type = dico["salome_parameters"]["job_type"]
+    self.salome_parameters.job_file = dico["salome_parameters"]["job_file"]
+    self.salome_parameters.pre_command = dico["salome_parameters"]["pre_command"]
+    self.salome_parameters.env_file = dico["salome_parameters"]["env_file"]
+    self.salome_parameters.in_files = dico["salome_parameters"]["in_files"]
+    self.salome_parameters.out_files = dico["salome_parameters"]["out_files"]
+    self.salome_parameters.work_directory = dico["salome_parameters"]["work_directory"]
+    self.salome_parameters.local_directory = dico["salome_parameters"]["local_directory"]
+    self.salome_parameters.result_directory = dico["salome_parameters"]["result_directory"]
+    self.salome_parameters.maximum_duration = dico["salome_parameters"]["maximum_duration"]
+    self.salome_parameters.queue = dico["salome_parameters"]["queue"]
+    self.salome_parameters.partition = dico["salome_parameters"]["partition"]
+    self.salome_parameters.exclusive = dico["salome_parameters"]["exclusive"]
+    self.salome_parameters.mem_per_cpu = dico["salome_parameters"]["mem_per_cpu"]
+    self.salome_parameters.wckey = dico["salome_parameters"]["wckey"]
+    self.salome_parameters.extra_params = dico["salome_parameters"]["extra_params"]
+    self.salome_parameters.resource_required.name = dico["salome_parameters"]["resource_required"]["name"]
+    self.salome_parameters.resource_required.hostname = dico["salome_parameters"]["resource_required"]["hostname"]
+    self.salome_parameters.resource_required.can_launch_batch_jobs = dico["salome_parameters"]["resource_required"]["can_launch_batch_jobs"]
+    self.salome_parameters.resource_required.can_run_containers = dico["salome_parameters"]["resource_required"]["can_run_containers"]
+    self.salome_parameters.resource_required.OS = dico["salome_parameters"]["resource_required"]["OS"]
+    self.salome_parameters.resource_required.nb_proc = dico["salome_parameters"]["resource_required"]["nb_proc"]
+    self.salome_parameters.resource_required.mem_mb = dico["salome_parameters"]["resource_required"]["mem_mb"]
+    self.salome_parameters.resource_required.cpu_clock = dico["salome_parameters"]["resource_required"]["cpu_clock"]
+    self.salome_parameters.resource_required.nb_node = dico["salome_parameters"]["resource_required"]["nb_node"]
+    self.salome_parameters.resource_required.nb_proc_per_node = dico["salome_parameters"]["resource_required"]["nb_proc_per_node"]

pydefx/plugins/jobexecutor.py

@@ -0,0 +1,130 @@
+import pydefx
+import os
+import pickle
+import time
+import traceback
+
+pydefx.forceNoSalomeServers()
+class Context:
+  def __init__(self):
+    self.launcher = pydefx.salome_proxy.getLauncher() # getLauncher()
+  pass
+
+class JobExecutor:
+  def __init__(self, config):
+    self.config = config
+
+  def initialize(self):
+    """ This is executed before the first evaluation.
+    Put here global processing needed by all the evaluations like the copy of
+    commun files.
+    """
+    # Copy the commun files to the root work directory
+    params = pydefx.Parameters() # global parameters
+    params.loadDict(self.config["params"])
+    # use a fake empty command.
+    # Using launcher to copy some files on the remote file system,
+    # without launching a job.
+    command = os.path.join(os.getcwd(), "empty.sh")
+    open(command, "w").close()
+    params.salome_parameters.job_file = command
+    params.salome_parameters.job_type = "command"
+    study_module = os.path.join(os.getcwd(), self.config["studymodule"]+".py")
+    infiles = list(params.salome_parameters.in_files)
+    params.salome_parameters.in_files = infiles + [study_module]
+    launcher = pydefx.salome_proxy.getLauncher()
+    job_id = launcher.createJob(params.salome_parameters)
+    launcher.exportInputFiles(job_id)
+
+  def evaluate(self, idx, point):
+    """ This is executed for every point to be evaluated.
+    """
+    context = Context()
+    error = None
+    out_values = None
+    try:
+      self.prepare(idx, point, context)
+      if self.noRunFound(idx, point, context):
+        self.runjob(idx, point, context)
+      error, out_values = self.getResult(context)
+    except Exception as e:
+      error = str(e)
+      traceback.print_exc()
+    return error, out_values
+
+  def prepare(self, idx, point, context):
+    """
+    Define local and remote work directory.
+    Define job script.
+    """
+    context.params = pydefx.Parameters()
+    context.params.loadDict(self.config["params"])
+    salome_parameters = context.params.salome_parameters
+    root_local_dir = salome_parameters.result_directory
+    root_remote_dir = salome_parameters.work_directory
+    input_files = [] # commun files are already copied to the root directory
+    point_name = "job_"+str(idx)
+    context.local_dir = os.path.join(root_local_dir, point_name)
+    point_remote_dir = os.path.join(root_remote_dir, point_name)
+    if not os.path.exists(context.local_dir):
+      os.mkdir(context.local_dir)
+    # export the point to a file
+    data_file_name = "idefixdata.csv"
+    data_file_path = os.path.join(context.local_dir, data_file_name)
+    with open(data_file_path, "w") as f:
+      # explicit dict convertion is needed for compatibility between python versions
+      f.write(repr(dict(point)))
+    input_files.append(data_file_path)
+
+    #command_path = os.path.join(root_local_dir, "command.py")
+    #salome_parameters.job_type = "command_salome"
+    #salome_parameters.job_file = command_path
+
+    salome_parameters.in_files = input_files
+    salome_parameters.out_files = ["idefixresult.txt", "idefixerror.txt"]
+    salome_parameters.work_directory = point_remote_dir
+    salome_parameters.result_directory = context.local_dir
+
+  def noRunFound(self, idx, point, context):
+    return True
+
+  def runjob(self, idx, point, context):
+    """
+    Create, launch and wait for the end of the job.
+    """
+    import random
+    sleep_delay = random.randint(5, 15) #10
+    #launcher = pydefx.salome_proxy.getLauncher()
+    launcher = context.launcher
+    context.job_id = launcher.createJob(context.params.salome_parameters)
+    launcher.launchJob(context.job_id)
+    jobState = launcher.getJobState(context.job_id)
+    while jobState=="QUEUED" or jobState=="IN_PROCESS" or jobState=="RUNNING" :
+      time.sleep(sleep_delay)
+      jobState = launcher.getJobState(context.job_id)
+
+  def getResult(self, context):
+    """
+    Check the job state, fetch the result file.
+    """
+    #launcher = pydefx.salome_proxy.getLauncher()
+    launcher = context.launcher
+    jobState = launcher.getJobState(context.job_id)
+    error=""
+    result=None
+    if jobState != "FINISHED" :
+      error = "Job has not finished correctly."
+    else:
+      launcher.getJobResults(context.job_id, "")
+      error_file = os.path.join(context.local_dir, "idefixerror.txt")
+      result_file = os.path.join(context.local_dir, "idefixresult.txt")
+      with open(error_file, "r") as f:
+        error = f.read()
+      with open(result_file, "r") as f:
+        result_str = f.read()
+        result = eval(result_str)
+
+    return error, result
+
+def createExecutor(config):
+  return JobExecutor(config)

pydefx/plugins/lightexecutor.py

@@ -0,0 +1,40 @@
+import os
+import pickle
+import time
+import traceback
+import subprocess
+
+class Context:
+  def __init__(self):
+    #self.launcher = pydefx.salome_proxy.getLauncher() # getLauncher()
+    pass
+
+class JobExecutor:
+  def __init__(self, config):
+    self.config = config
+
+  def initialize(self):
+    """ This is executed before the first evaluation.
+    Put here global processing needed by all the evaluations like the copy of
+    commun files.
+    """
+    pass
+
+  def evaluate(self, idx, point):
+    """ This is executed for every point to be evaluated.
+    """
+    context = Context()
+    error = None
+    out_values = None
+    studymodule=self.config["studymodule"]
+    import importlib
+    try:
+      idefixstudy=importlib.import_module(studymodule)
+      out_values=idefixstudy._exec(**point)
+    except Exception as e:
+      error=str(e)
+      traceback.print_exc()
+    return error, out_values
+
+def createExecutor(config):
+  return JobExecutor(config)

pydefx/plugins/localexecutor.py

@@ -0,0 +1,81 @@
+import os
+import pickle
+import time
+import traceback
+import subprocess
+
+class Context:
+  def __init__(self):
+    pass
+
+class JobExecutor:
+  def __init__(self, config):
+    self.config = config
+
+  def initialize(self):
+    """ This is executed before the first evaluation.
+    Put here global processing needed by all the evaluations like the copy of
+    commun files.
+    """
+    pointeval = os.path.join(os.getcwd(), "pointeval.py")
+    os.chmod(pointeval, 0o755)
+
+  def evaluate(self, idx, point):
+    """ This is executed for every point to be evaluated.
+    """
+    context = Context()
+    error = None
+    out_values = None
+    try:
+      self.prepare(idx, point, context)
+      if self.noRunFound(idx, point, context):
+        self.runjob(idx, point, context)
+      error, out_values = self.getResult(context)
+    except Exception as e:
+      error = str(e)
+      traceback.print_exc()
+    return error, out_values
+
+  def prepare(self, idx, point, context):
+    """
+    Define local and remote work directory.
+    Define job script.
+    """
+    root_dir = os.getcwd()
+    point_name = "job_"+str(idx)
+    context.local_dir = os.path.join(root_dir, point_name)
+    if not os.path.exists(context.local_dir):
+      os.mkdir(context.local_dir)
+    # export the point to a file
+    data_file_name = "idefixdata.csv"
+    data_file_path = os.path.join(context.local_dir, data_file_name)
+    with open(data_file_path, "w") as f:
+      # explicit dict convertion is needed for compatibility between python versions
+      f.write(repr(dict(point)))
+
+  def noRunFound(self, idx, point, context):
+    return True
+
+  def runjob(self, idx, point, context):
+    """
+    Create, launch and wait for the end of the job.
+    """
+    pointeval = os.path.join(os.getcwd(), "pointeval.py")
+    return_code = subprocess.check_call(pointeval, shell=True, cwd=context.local_dir)
+
+  def getResult(self, context):
+    """
+    Check the job state, fetch the result file.
+    """
+    error_file = os.path.join(context.local_dir, "idefixerror.txt")
+    result_file = os.path.join(context.local_dir, "idefixresult.txt")
+    with open(error_file, "r") as f:
+      error = f.read()
+    with open(result_file, "r") as f:
+      result_str = f.read()
+      result = eval(result_str)
+
+    return error, result
+
+def createExecutor(config):
+  return JobExecutor(config)

pydefx/plugins/mainjob.py

@@ -0,0 +1,56 @@
+#! /usr/bin/env python3
+import json
+import importlib
+from multiprocessing import Pool
+import traceback
+
+class StartJob:
+  def __init__(self, executor):
+    self.executor = executor
+
+  def __call__(self, idx, in_values):
+    error=None
+    out_values=None
+    try:
+      error, out_values = self.executor.evaluate(idx, in_values)
+    except Exception as e:
+      error=str(e)
+      traceback.print_exc()
+    return idx, in_values, out_values, error
+
+class TerminateJob:
+  def __init__(self, manager):
+    self.manager = manager
+
+  def __call__(self, result):
+    # without try statement we may experience deadlock in case of error.
+    try:
+      idx, in_values, out_values, error = result
+      if not error:
+        error = None
+      self.manager.addResult(idx, in_values, out_values, error)
+    except Exception as e:
+      traceback.print_exc()
+
+if __name__ == '__main__':
+  with open("idefixconfig.json", "r") as f:
+    config = json.load(f)
+  plugin_module = importlib.import_module(config["plugin"])
+  executor = plugin_module.createExecutor(config)
+  # global initialization - commun work for every evaluation.
+  executor.initialize()
+
+  itModuleName = config["sampleIterator"]
+  itModule = importlib.import_module(itModuleName)
+  sampleManager = itModule.SampleIterator()
+  sampleManager.writeHeaders()
+
+  nbbranches=config["nbbranches"]
+  pool = Pool(nbbranches)
+  runPoint = StartJob(executor)
+  endOk = TerminateJob(sampleManager)
+  for point in sampleManager:
+    pool.apply_async(runPoint, point, callback=endOk)
+  pool.close()
+  pool.join()
+  sampleManager.terminate()

pydefx/plugins/pointeval.py

@@ -0,0 +1,39 @@
+#! /usr/bin/env python3
+import traceback
+import os
+
+data_file_name = "idefixdata.csv"
+study_module = "idefixstudy.py"
+error_result = "idefixerror.txt"
+value_result = "idefixresult.txt"
+traceback_result = "idefixtraceback.txt"
+
+with open(data_file_name, "r") as f:
+  values = f.read()
+inputvals = eval(values)
+
+error=""
+result=None
+old_dir = os.getcwd()
+
+try:
+  os.chdir("..") # go to commun root directory
+  with open(study_module, "r") as study_file:
+    study_string = study_file.read()
+  exec(study_string)
+  result = _exec(**inputvals)
+except Exception as e:
+  error=str(e)
+  if not error :
+    error = "Exception " + repr(e)
+  os.chdir(old_dir) # back to the current case job directory
+  with open(traceback_result, "w") as f:
+    traceback.print_exc(file=f)
+
+os.chdir(old_dir) # back to the current case job directory
+
+with open(error_result, "w") as f:
+  f.write(error)
+
+with open(value_result, "w") as f:
+  f.write(repr(result))

pydefx/plugins/srunexecutor.py

@@ -0,0 +1,87 @@
+import os
+import pickle
+import time
+import traceback
+import subprocess
+
+class Context:
+  def __init__(self):
+    #self.launcher = pydefx.salome_proxy.getLauncher() # getLauncher()
+    pass
+
+class JobExecutor:
+  def __init__(self, config):
+    self.config = config
+
+  def initialize(self):
+    """
+    Execute prescript.
+    """
+    pointeval = os.path.join(os.getcwd(), "pointeval.py")
+    os.chmod(pointeval, 0o755)
+
+  def evaluate(self, idx, point):
+    """ This is executed for every point to be evaluated.
+    """
+    context = Context()
+    error = None
+    out_values = None
+    try:
+      self.prepare(idx, point, context)
+      if self.noRunFound(idx, point, context):
+        self.runjob(idx, point, context)
+      error, out_values = self.getResult(context)
+    except Exception as e:
+      error = str(e)
+      traceback.print_exc()
+    return error, out_values
+
+  def prepare(self, idx, point, context):
+    """
+    Define local and remote work directory.
+    Define job script.
+    """
+    root_dir = os.getcwd()
+    point_name = "job_"+str(idx)
+    context.local_dir = os.path.join(root_dir, point_name)
+    if not os.path.exists(context.local_dir):
+      os.mkdir(context.local_dir)
+    # export the point to a file
+    data_file_name = "idefixdata.csv"
+    data_file_path = os.path.join(context.local_dir, data_file_name)
+    with open(data_file_path, "w") as f:
+      # explicit dict convertion is needed for compatibility between python versions
+      f.write(repr(dict(point)))
+
+  def noRunFound(self, idx, point, context):
+    return True
+
+  def runjob(self, idx, point, context):
+    """
+    Create, launch and wait for the end of the job.
+    """
+    # srun
+    ntasks = self.config["tasksPerEval"]
+    pointeval = os.path.join(os.getcwd(), "pointeval.py")
+    command = "srun --ntasks={} --nodes=1 --chdir={} {} ".format(
+                                                             str(ntasks),
+                                                             context.local_dir,
+                                                             pointeval)
+    return_code = subprocess.call(command, shell=True)
+
+  def getResult(self, context):
+    """
+    Check the job state, fetch the result file.
+    """
+    error_file = os.path.join(context.local_dir, "idefixerror.txt")
+    result_file = os.path.join(context.local_dir, "idefixresult.txt")
+    with open(error_file, "r") as f:
+      error = f.read()
+    with open(result_file, "r") as f:
+      result_str = f.read()
+      result = eval(result_str)
+
+    return error, result
+
+def createExecutor(config):
+  return JobExecutor(config)