PyFemtet 0.3.3__tar.gz → 0.3.5__tar.gz

{pyfemtet-0.3.3 → pyfemtet-0.3.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pyfemtet
-Version: 0.3.3
+Version: 0.3.5
 Summary: Design parameter optimization using Femtet.
 Home-page: https://github.com/pyfemtet/pyfemtet
 License: BSD-3-Clause

pyfemtet-0.3.5/pyfemtet/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.3.5"

{pyfemtet-0.3.3 → pyfemtet-0.3.5}/pyfemtet/dispatch_extensions.py

@@ -4,6 +4,7 @@ import os
 from time import time, sleep
 from multiprocessing import current_process
 
+from tqdm import tqdm
 import psutil
 from dask.distributed import Lock
 
@@ -15,6 +16,7 @@ from femtetutils import util
 
 from multiprocessing.context import BaseContext, SpawnProcess, _concrete_contexts
 from multiprocessing.process import _children, _cleanup
+from multiprocessing.managers import SyncManager
 
 import logging
 from .logger import get_logger
@@ -138,11 +140,16 @@ def _get_pids(process_name):
     return pids
 
 
-def launch_and_dispatch_femtet(timeout=DISPATCH_TIMEOUT) -> Tuple[IFemtet, int]:
+def _get_subprocess_log_prefix():
+    return f'({current_process().name}) '
+
+
+def launch_and_dispatch_femtet(timeout=DISPATCH_TIMEOUT, strict_pid_specify=True) -> Tuple[IFemtet, int]:
     """Launch Femtet by new process and connect to it.
 
     Args:
         timeout (int or float, optional): Seconds to wait for connection. Defaults to DISPATCH_TIMEOUT.
+        strict_pid_specify (bool): If True, try to connect the launched femtet process strictly. If False, launch new process but try to connect any connectable femtet.
 
     Raises:
         FemtetNotFoundException: Launched Femtet is not found for some reason (i.e. failed to launch Femtet).
@@ -151,11 +158,26 @@ def launch_and_dispatch_femtet(timeout=DISPATCH_TIMEOUT) -> Tuple[IFemtet, int]:
     Returns:
         Tuple[IFemtet, int]:
     """
+    # launch femtet
     util.execute_femtet()
     pid = util.get_last_executed_femtet_process_id()
     logger.debug(f'Target pid is {pid}.')
-    sleep(5)
-    Femtet, pid = dispatch_specific_femtet(pid, timeout)
+    for _ in tqdm(range(5), 'wait for launch femtet...'):
+        sleep(1)
+
+    # dispatch femtet
+    if strict_pid_specify:
+        Femtet, pid = dispatch_specific_femtet(pid, timeout)
+    else:
+        # worker process なら排他処理する
+        try:
+            with Lock('simply-dispatch-femtet'):
+                Femtet, pid = dispatch_femtet()
+        except RuntimeError as e:
+            if "distributed.lock.Lock" in str(e):
+                Femtet, pid = dispatch_femtet()
+            else:
+                raise e
     return Femtet, pid
 
 
@@ -207,9 +229,6 @@ def dispatch_femtet(timeout=DISPATCH_TIMEOUT, subprocess_log_prefix='') -> Tuple
     return Femtet, pid
 
 
-def _get_subprocess_log_prefix():
-    return f'({current_process().name}) '
-
 
 def _block_other_femtets(
         target_pid,
@@ -386,10 +405,11 @@ def dispatch_specific_femtet_core(pid, timeout=DISPATCH_TIMEOUT) -> Tuple[IFemte
     if not (pid in pids):
         raise FemtetNotFoundException(f"Femtet (pid = {pid}) doesn't exist.")
 
+    logger.info('Searching specific Femtet...')
+
     # 子プロセスの準備
     # with Manager() as manager:
-    from multiprocessing.managers import SyncManager
-    m = SyncManager(ctx=_NestableSpawnContext())
+    m = SyncManager(ctx=_NestableSpawnContext())  # このへんが時間のかかる処理
     m.start()
     with m as manager:
         # フラグの準備
@@ -402,7 +422,7 @@ def dispatch_specific_femtet_core(pid, timeout=DISPATCH_TIMEOUT) -> Tuple[IFemte
         # 目的以外の Femtet をブロックする子プロセスを開始
         logger.debug('Start subprocess to block Femtet other than target pid.')
         processes = []
-        for subprocess_id in range(len(pids)):
+        for subprocess_id in tqdm(range(len(pids)), 'Specifying connection...'):
             p = _NestableSpawnProcess(
             # p = Process(
                 target=_block_other_femtets,

{pyfemtet-0.3.3 → pyfemtet-0.3.5}/pyfemtet/logger.py

@@ -12,8 +12,10 @@ def _get_worker_name_as_prefix():
     name = '(Main) '
     try:
         worker = get_worker()
-        # address = worker.address
-        name = f'(Worker:{worker.name}) '
+        if isinstance(worker.name, str):  # local なら index, cluster なら tcp address
+            name = f'({worker.name}) '
+        else:
+            name = f'(Sub{worker.name}) '
     except ValueError:
         pass
     return name

{pyfemtet-0.3.3 → pyfemtet-0.3.5}/pyfemtet/opt/base.py

@@ -1,5 +1,5 @@
 from abc import ABC, abstractmethod
-from typing import Callable, List, Tuple, Iterable, Any
+from typing import List, Iterable
 
 import os
 import datetime
@@ -65,13 +65,13 @@ def generate_lhs(bounds: List[List[float]], seed: int or None = None) -> np.ndar
     LIMIT = 100
 
     def is_prime(p):
-        for i in range(2, p):
-            if p % i == 0:
+        for j in range(2, p):
+            if p % j == 0:
                 return False
         return True
 
-    def get_prime(minimum):
-        for p in range(minimum, LIMIT):
+    def get_prime(_minimum):
+        for p in range(_minimum, LIMIT):
             if is_prime(p):
                 return p
 
@@ -252,7 +252,7 @@ class Objective(Function):
         super().__init__(fun, name, args, kwargs)
 
     def convert(self, value: float):
-        """Converts an evaluation value to the value of user-defined objective function based on the specified direction.
+        """Converts an evaluation value to the value of objective function based on the specified direction.
 
         When direction is `'minimize'`, ``value`` is calculated.
         When direction is `'maximize'`, ``-value`` is calculated.
@@ -329,9 +329,8 @@ class History:
     """Class for managing the history of optimization results.
 
     Attributes:
-        path (str): The path to the history file.
-        _actor_data (HistoryDfCore): The distributed DataFrame object for storing actor data.
-        data (pd.DataFrame): The DataFrame object for storing the entire history.
+        path (str): The path to the history csv file.
+        is_restart (bool): The main session is restarted or not.
         param_names (list): The names of the parameters in the study.
         obj_names (list): The names of the objectives in the study.
         cns_names (list): The names of the constraints in the study.
@@ -350,13 +349,15 @@ class History:
         self.is_restart = False
         self._future = client.submit(HistoryDfCore, actor=True)
         self._actor_data = self._future.result()
+        self.tmp_data = pd.DataFrame()
         self.param_names = []
         self.obj_names = []
         self.cns_names = []
 
         # path が存在すれば dataframe を読み込む
         if os.path.isfile(self.path):
-            self.actor_data = pd.read_csv(self.path, encoding='shift-jis')
+            self.tmp_data = pd.read_csv(self.path, encoding='shift-jis')
+            self.actor_data = self.tmp_data
             self.is_restart = True
 
     @property
@@ -406,6 +407,8 @@ class History:
                 raise Exception(f'読み込んだ history と問題の設定が異なります. \n\n読み込まれた設定:\n{list(self.actor_data.columns)}\n\n現在の設定:\n{columns}')
 
         else:
+            for column in columns:
+                self.tmp_data[column] = None
             # actor_data は actor 経由の getter property なので self.data[column] = ... とやっても
             # actor には変更が反映されない. 以下同様
             tmp = self.actor_data
@@ -445,27 +448,25 @@ class History:
         with Lock('calc-history'):
             # append
             if len(self.actor_data) == 0:
-                self.actor_data = pd.DataFrame([row], columns=self.actor_data.columns)
+                self.tmp_data = pd.DataFrame([row], columns=self.actor_data.columns)
             else:
-                tmp = self.actor_data
-                tmp.loc[len(tmp)] = row
-                self.actor_data = tmp
+                self.tmp_data = self.actor_data
+                self.tmp_data.loc[len(self.tmp_data)] = row
 
             # calc
-            tmp = self.actor_data
-            tmp['trial'] = np.arange(len(tmp)) + 1  # 1 始まり
-            self.actor_data = tmp
-            self._calc_non_domi(objectives)
-            self._calc_hypervolume(objectives)
+            self.tmp_data['trial'] = np.arange(len(self.tmp_data)) + 1  # 1 始まり
+            self._calc_non_domi(objectives)  # update self.tmp_data
+            self._calc_hypervolume(objectives)  # update self.tmp_data
+            self.actor_data = self.tmp_data
 
     def _calc_non_domi(self, objectives):
 
         # 目的関数の履歴を取り出してくる
-        solution_set = self.actor_data[self.obj_names].copy()
+        solution_set = self.tmp_data[self.obj_names]
 
         # 最小化問題の座標空間に変換する
         for name, objective in objectives.items():
-            solution_set[name] = solution_set[name].map(objective.convert)
+            solution_set.loc[:, name] = solution_set[name].map(objective.convert)
 
         # 非劣解の計算
         non_domi = []
@@ -473,15 +474,11 @@ class History:
             non_domi.append((row > solution_set).product(axis=1).sum(axis=0) == 0)
 
         # 非劣解の登録
-        tmp = self.actor_data
-        tmp['non_domi'] = non_domi
-        self.actor_data = tmp
-
-        del solution_set
+        self.tmp_data['non_domi'] = non_domi
 
     def _calc_hypervolume(self, objectives):
-        # プロセス間の競合を避けるためローカルにコピーする
-        df = self.actor_data.copy()
+        # タイピングが面倒
+        df = self.tmp_data
 
         # パレート集合の抽出
         idx = df['non_domi'].values
@@ -540,15 +537,12 @@ class History:
         # dominated の行に対して、上に見ていって
         # 最初に見つけた non-domi 行の hypervolume の値を割り当てます
         for i in range(len(df)):
-            if df.loc[i, 'non_domi'] == False:
+            if not df.loc[i, 'non_domi']:
                 try:
                     df.loc[i, 'hypervolume'] = df.loc[:i][df.loc[:i]['non_domi']].iloc[-1]['hypervolume']
                 except IndexError:
                     df.loc[i, 'hypervolume'] = 0
 
-        # ローカルの df を worker に戻す
-        self.actor_data = df
-
 
 class OptimizationStatusActor:
     status_int = -1
@@ -568,35 +562,41 @@ class OptimizationStatus:
     UNDEFINED = -1
     INITIALIZING = 0
     SETTING_UP = 10
-    LAUNCHING = 20
-    WAIT_1ST = 25
+    LAUNCHING_FEM = 20
+    WAIT_OTHER_WORKERS = 22
+    # WAIT_1ST = 25
     RUNNING = 30
     INTERRUPTING = 40
     TERMINATED = 50
     TERMINATE_ALL = 60
 
-    def __init__(self, client):
+    def __init__(self, client, name='entire'):
         self._future = client.submit(OptimizationStatusActor, actor=True)
         self._actor = self._future.result()
+        self.name = name
         self.set(self.INITIALIZING)
 
     @classmethod
-    def const_to_str(self, status_const):
-        if status_const == self.UNDEFINED: return 'Undefined'
-        if status_const == self.INITIALIZING: return 'Initializing'
-        if status_const == self.SETTING_UP: return 'Setting up'
-        if status_const == self.LAUNCHING: return 'Launching FEM processes'
-        if status_const == self.WAIT_1ST: return 'Running and waiting for 1st FEM result.'
-        if status_const == self.RUNNING: return 'Running'
-        if status_const == self.INTERRUPTING: return 'Interrupting'
-        if status_const == self.TERMINATED: return 'Terminated'
-        if status_const == self.TERMINATE_ALL: return 'Terminate_all'
+    def const_to_str(cls, status_const):
+        if status_const == cls.UNDEFINED: return 'Undefined'
+        if status_const == cls.INITIALIZING: return 'Initializing'
+        if status_const == cls.SETTING_UP: return 'Setting up'
+        if status_const == cls.LAUNCHING_FEM: return 'Launching FEM processes'
+        if status_const == cls.WAIT_OTHER_WORKERS: return 'Waiting for launching other processes'
+        # if status_const == cls.WAIT_1ST: return 'Running and waiting for 1st FEM result.'
+        if status_const == cls.RUNNING: return 'Running'
+        if status_const == cls.INTERRUPTING: return 'Interrupting'
+        if status_const == cls.TERMINATED: return 'Terminated'
+        if status_const == cls.TERMINATE_ALL: return 'Terminate_all'
 
     def set(self, status_const):
-        # self._actor.status_int = status_const
-        # self._actor.status = self.const_to_str(status_const)
         self._actor.set(status_const, self.const_to_str(status_const)).result()
-        logger.info(f'---{self.const_to_str(status_const)}---')
+        msg = f'---{self.const_to_str(status_const)}---'
+        if (status_const == self.INITIALIZING) and (self.name != 'entire'):
+            msg += f' (for Worker {self.name})'
+        if self.name == 'entire':
+            msg = '(entire) ' + msg
+        logger.info(msg)
 
     def get(self):
         return self._actor.status_int
@@ -614,8 +614,9 @@ class AbstractOptimizer(ABC):
         self.parameters = pd.DataFrame()
         self.objectives = dict()
         self.constraints = dict()
-        self.status = None  # shared
-        self.history = None  # shared
+        self.entire_status = None  # actor
+        self.history = None  # actor
+        self.worker_status = None  # actor
         self.message = ''
         self.seed = None
         self.timeout = None
@@ -623,9 +624,7 @@ class AbstractOptimizer(ABC):
         self.is_cluster = False
 
     def f(self, x):
-
-        if self.status.get() == OptimizationStatus.LAUNCHING:
-            self.status.set(OptimizationStatus.WAIT_1ST)
+        # interruption の実装は具象クラスに任せる
 
         # x の更新
         self.parameters['value'] = x
@@ -654,14 +653,13 @@ class AbstractOptimizer(ABC):
             self.message
         )
 
-        if self.status.get() == OptimizationStatus.WAIT_1ST:
-            self.status.set(OptimizationStatus.RUNNING)
-
         logger.debug('history.record end')
         return np.array(y), np.array(_y), np.array(c)
 
-    def set_fem(self):
-        self.fem = self.fem_class(**self.fem_kwargs)
+    def set_fem(self, skip_reconstruct=False):
+        # restore fem
+        if not skip_reconstruct:
+            self.fem = self.fem_class(**self.fem_kwargs)
 
         # COM 定数の restore
         for obj in self.objectives.values():
@@ -694,12 +692,66 @@ class AbstractOptimizer(ABC):
         else:
             raise ValueError('get_parameter() got invalid format: {format}')
 
-    def _main(self, subprocess_idx):
-        self.set_fem()
-        self.main(subprocess_idx)
+    def check_interruption(self):
+        if self.entire_status.get() == OptimizationStatus.INTERRUPTING:
+            self.worker_status.set(OptimizationStatus.INTERRUPTING)
+            self.finalize()
+            return True
+        else:
+            return False
+
+    def finalize(self):
+        del self.fem
+        self.worker_status.set(OptimizationStatus.TERMINATED)
+
+    def _main(
+            self,
+            subprocess_idx,
+            worker_status_list,
+            wait_setup,
+            skip_set_fem=False,
+    ) -> None:
+
+        # 自分の worker_status の取得
+        self.worker_status = worker_status_list[subprocess_idx]
+        self.worker_status.set(OptimizationStatus.LAUNCHING_FEM)
+
+        if self.check_interruption():
+            return None
+
+        # set_fem をはじめ、終了したらそれを示す
+        if not skip_set_fem:  # なくても動く？？
+            self.set_fem()
+        self.worker_status.set(OptimizationStatus.WAIT_OTHER_WORKERS)
+
+        # wait_setup or not
+        if wait_setup:
+            while True:
+                if self.check_interruption():
+                    return None
+                # 他のすべての worker_status が wait 以上になったら break
+                if all([ws.get() >= OptimizationStatus.WAIT_OTHER_WORKERS for ws in worker_status_list]):
+                    break
+                sleep(1)
+        else:
+            if self.check_interruption():
+                return None
+
+        # set status running
+        if self.entire_status.get() < OptimizationStatus.RUNNING:
+            self.entire_status.set(OptimizationStatus.RUNNING)
+        self.worker_status.set(OptimizationStatus.RUNNING)
+
+        # run and finalize
+        try:
+            self.main(subprocess_idx)
+        finally:
+            self.finalize()
+
+        return None
 
     @abstractmethod
-    def main(self, subprocess_idx: int):
+    def main(self, subprocess_idx: int = 0) -> None:
         """Called from dask worker."""
         pass
 
@@ -731,7 +783,7 @@ class OptunaOptimizer(AbstractOptimizer):
     def _objective(self, trial):
 
         # 中断の確認 (FAIL loop に陥る対策)
-        if self.status.get() == OptimizationStatus.INTERRUPTING:
+        if self.entire_status.get() == OptimizationStatus.INTERRUPTING:
             trial.study.stop()  # 現在実行中の trial を最後にする
             return None  # set TrialState FAIL
 
@@ -772,7 +824,7 @@ class OptunaOptimizer(AbstractOptimizer):
             print(self.get_parameter('dict'))
 
             # 中断の確認 (解析中に interrupt されている場合対策)
-            if self.status.get() == OptimizationStatus.INTERRUPTING:
+            if self.entire_status.get() == OptimizationStatus.INTERRUPTING:
                 trial.study.stop()  # 現在実行中の trial を最後にする
                 return None  # set TrialState FAIL
 
@@ -789,7 +841,7 @@ class OptunaOptimizer(AbstractOptimizer):
         trial.set_user_attr('constraint', _c)
 
         # 中断の確認 (解析中に interrupt されている場合対策)
-        if self.status.get() == OptimizationStatus.INTERRUPTING:
+        if self.entire_status.get() == OptimizationStatus.INTERRUPTING:
             trial.study.stop()  # 現在実行中の trial を最後にする
             return None  # set TrialState FAIL
 
@@ -867,14 +919,14 @@ class OptunaOptimizer(AbstractOptimizer):
         # if is_restart, load study
         else:
             if not os.path.exists(storage_path):
-                raise FileNotFoundError(f'{storage_path} が見つかりません。.db ファイルは .csv ファイルと同じフォルダに生成されます。クラスター解析の場合は、スケジューラを起動したフォルダに生成されます。')
+                msg = f'{storage_path} が見つかりません。'
+                msg += '.db ファイルは .csv ファイルと同じフォルダに生成されます。'
+                msg += 'クラスター解析の場合は、スケジューラを起動したフォルダに生成されます。'
+                raise FileNotFoundError(msg)
             self.storage = optuna.integration.dask.DaskStorage(
                 f'sqlite:///{storage_path}',
             )
 
-
-
-
     def add_init_parameter(
             self,
             parameter: dict or Iterable,
@@ -895,7 +947,7 @@ class OptunaOptimizer(AbstractOptimizer):
             name = f'additional initial ({name})'
         self.additional_initial_parameter.append([parameter, name])
 
-    def main(self, subprocess_idx):
+    def main(self, subprocess_idx=0):
         """Dask worker から呼ばれる関数"""
 
         # (re)set random seed
@@ -925,27 +977,20 @@ class OptunaOptimizer(AbstractOptimizer):
             callbacks=self.optimize_callbacks,
         )
 
-        # finalize
-        del self.fem
-
 
 class FEMOpt:
     """Base class to control FEM interface and optimizer.
 
     Attributes:
-        fem (FEMInterface): The finite element method interface.
-        history_path (str): The path to the history (.csv) file .
-        ipv (InterprocessVariables): The interprocess variables.
-        parameters (pd.DataFrame): The DataFrame object for storing the parameters.
-        objectives (dict): The dictionary of objective functions.
-        constraints (dict): The dictionary of constraint functions.
-        history (History): The history of optimization results.
-        monitor (Monitor): The monitor object for visualization and monitoring.
-        monitor_thread: Thread object for monitor server.
-        seed (int or None): The random seed for reproducibility.
-        message(str) : Additional information or messages related to the optimization process
-        obj_values ([float]): A list to store objective values during optimization
-        cns_values ([float]): A list to store constraint values during optimization
+        fem (FEMInterface): The interface of FEM system.
+        client (Client): Dask client. For detail, see dask documentation.
+        scheduler_address (str or None): Dask scheduler address. If None, LocalCluster will be used.
+        status (OptimizationStatus): Entire process status. This contains dask actor.
+        history(History): History of optimization process. This contains dask actor.
+        history_path (str): The path to the history (.csv) file.
+        worker_status_list([OptimizationStatus]): Process status of each dask worker.
+        monitor_process_future(Future): Future of monitor server process. This is dask future.
+        monitor_server_kwargs(dict): Monitor server parameter. Currently, the valid arguments are hostname and port.
 
     """
 
@@ -959,7 +1004,7 @@ class FEMOpt:
         """Initializes an FEMOpt instance.
 
         Args:
-            fem (FEMInterface, optional): The finite element method interface. Defaults to None. If None, automattically set to FemtetInterface.
+            fem (FEMInterface, optional): The finite element method interface. Defaults to None. If None, automatically set to FemtetInterface.
             opt (AbstractOptimizer):
             history_path (str, optional): The path to the history file. Defaults to None. If None, '%Y_%m_%d_%H_%M_%S.csv' is created in current directory.
             scheduler_address (str or None): If cluster processing, set this parameter like ``"tcp://xxx.xxx.xxx.xxx:xxxx"``.
@@ -986,10 +1031,12 @@ class FEMOpt:
 
         # メンバーの宣言
         self.client = None
-        self.status = None
-        self.history = None
+        self.status = None  # actor
+        self.history = None  # actor
+        self.worker_status_list = None  # [actor]
         self.monitor_process_future = None
         self.monitor_server_kwargs = dict()
+        self.monitor_process_worker_name = None
 
     # multiprocess 時に pickle できないオブジェクト参照の削除
     def __getstate__(self):
@@ -1021,7 +1068,7 @@ class FEMOpt:
         
         Args:
             name (str): The name of the parameter.
-            initial_value (float or None, optional): The initial value of the parameter. Defaults to None. If None, try to get inittial value from FEMInterface.
+            initial_value (float or None, optional): The initial value of the parameter. Defaults to None. If None, try to get initial value from FEMInterface.
             lower_bound (float or None, optional): The lower bound of the parameter. Defaults to None. However, this argument is required for some algorithms.
             upper_bound (float or None, optional): The upper bound of the parameter. Defaults to None. However, this argument is required for some algorithms.
             memo (str, optional): Additional information about the parameter. Defaults to ''.
@@ -1198,16 +1245,16 @@ class FEMOpt:
             n_trials=None,
             n_parallel=1,
             timeout=None,
+            wait_setup=True,
     ):
         """Runs the main optimization process.
 
         Args:
-            n_trials (int or None): The number of trials. Defaults to None.
-            n_parallel (int): The number of parallel processes. Defaults to 1.
-            timeout (float or None): The maximum amount of time in seconds that each trial can run. Defaults to None.
-            method (str): The optimization method to use. Defaults to 'TPE'.
-            **setup_kwargs: Additional keyword arguments for setting up the optimization process.
-        
+            n_trials (int or None, optional): The number of trials. Defaults to None.
+            n_parallel (int, optional): The number of parallel processes. Defaults to 1.
+            timeout (float or None, optional): The maximum amount of time in seconds that each trial can run. Defaults to None.
+            wait_setup (bool, optional): Wait for all workers launching FEM system. Defaults to True.
+
         Tip:
             If setup_monitor_server() is not executed, a local server for monitoring will be started at localhost:8080.
 
@@ -1224,37 +1271,46 @@ class FEMOpt:
 
         """
 
+        # 共通引数
+        self.opt.n_trials = n_trials
+        self.opt.timeout = timeout
 
-        # dask メンバーの設定
+        # クラスターの設定
         self.opt.is_cluster = self.scheduler_address is not None
         if self.opt.is_cluster:
             # 既存のクラスターに接続
             logger.info('Connecting to existing cluster.')
             self.client = Client(self.scheduler_address)
-
         else:
+            # ローカルクラスターを構築
             logger.info('Launching single machine cluster. This may take tens of seconds.')
             cluster = LocalCluster(processes=True)
             self.client = Client(cluster, direct_to_workers=False)
             self.scheduler_address = self.client.scheduler.address
 
+        # タスクを振り分ける worker を指定
+        subprocess_indices = list(range(n_parallel))
+        if not self.opt.is_cluster:
+            subprocess_indices = subprocess_indices[1:]
+        worker_addresses = list(self.client.nthreads().keys())
+        assert max(subprocess_indices) <= len(worker_addresses)-1, f'コア数{len(worker_addresses)}は不足しています。'
+        worker_addresses = worker_addresses[:len(range(n_parallel))]  # TODO: ノードごとに適度に振り分ける
+        if not self.opt.is_cluster:
+            worker_addresses[0] = 'Main'
+
         # monitor 用 worker を起動
         logger.info('Launching monitor server. This may take a few seconds.')
         self.monitor_process_worker_name = datetime.datetime.now().strftime("Monitor-%Y%m%d-%H%M%S")
         cmd = f'dask worker {self.client.scheduler.address} --name {self.monitor_process_worker_name} --no-nanny'
-        worker_addresses = list(self.client.has_what().keys())
-        Popen(cmd, shell=True)  #, stdout=PIPE) --> cause stream error
+        current_n_workers = len(self.client.nthreads().keys())
+        Popen(cmd, shell=True)  # , stdout=PIPE) --> cause stream error
 
         # monitor 用 worker が増えるまで待つ
-        while not (len(self.client.has_what()) > len(worker_addresses)):
-            sleep(1)
-        new_worker_addresses = list(self.client.has_what().keys())
-        launched_worker_addresses = [w_adrs for w_adrs in new_worker_addresses if not w_adrs in worker_addresses]
-        assert len(launched_worker_addresses) == 1
-        launched_worker_address = launched_worker_addresses[0]
+        self.client.wait_for_workers(n_workers=current_n_workers+1)
 
         # actor の設定
         self.status = OptimizationStatus(self.client)
+        self.worker_status_list = [OptimizationStatus(self.client, name) for name in worker_addresses]  # tqdm 検討
         self.status.set(OptimizationStatus.SETTING_UP)
         self.history = History(self.history_path, self.client)
         self.history.init(
@@ -1263,17 +1319,13 @@ class FEMOpt:
             list(self.opt.constraints.keys()),
         )
 
-        # 共通引数
-        self.opt.n_trials = n_trials
-        self.opt.timeout = timeout
-
         # fem
         self.fem.setup_before_parallel(self.client)
 
         # opt
         self.opt.fem_class = type(self.fem)
         self.opt.fem_kwargs = self.fem.kwargs
-        self.opt.status = self.status
+        self.opt.entire_status = self.status
         self.opt.history = self.history
         self.opt.setup_before_parallel()
 
@@ -1282,39 +1334,42 @@ class FEMOpt:
             start_monitor_server_forever,
             self.history, self.status,  # args
             **self.monitor_server_kwargs,  # kwargs
-            workers=launched_worker_address,  # if invalid arg,
+            workers=self.monitor_process_worker_name,  # if invalid arg,
             allow_other_workers=False
         )
 
         # クラスターでの計算開始
-        self.status.set(OptimizationStatus.LAUNCHING)
+        self.status.set(OptimizationStatus.LAUNCHING_FEM)
         start = time()
-        subprocess_indices = list(range(n_parallel))
+        calc_futures = self.client.map(
+            self.opt._main,
+            subprocess_indices,
+            [self.worker_status_list]*len(subprocess_indices),
+            [wait_setup]*len(subprocess_indices),
+            workers=worker_addresses,
+            allow_other_workers=False,
+        )
 
-        if self.opt.is_cluster:
-            # FIXME: monitor-server-process には割り当てないようにする
-            worker_addresses = list(self.client.nthreads().keys())
-            worker_addresses.remove(launched_worker_address)
-            calc_futures = self.client.map(
-                self.opt._main,
-                subprocess_indices,
-                workers=worker_addresses,
-                allow_other_workers=False,
-            )
+        t_main = None
+        if not self.opt.is_cluster:
+            # ローカルプロセスでの計算(opt._main 相当の処理)
+            subprocess_idx = 0
 
-        else:
-            # ローカルなら既存の fem をひとつ使う
-            calc_futures = self.client.map(self.opt._main, subprocess_indices[1:])
-
-            # ローカルプロセスでの計算開始
-            # constants の restore （map の後なら大丈夫）
-            # TODO: 最適化中に cluster を scale する場合はここに気を付ける
-            for obj in self.opt.objectives.values():
-                obj._restore_constants()
-            for cns in self.opt.constraints.values():
-                cns._restore_constants()
+            # set_fem
             self.opt.fem = self.fem
-            t_main = Thread(target=self.opt.main, args=(subprocess_indices[0],))
+            self.opt.set_fem(skip_reconstruct=True)
+
+            t_main = Thread(
+                target=self.opt._main,
+                args=(
+                    subprocess_idx,
+                    self.worker_status_list,
+                    wait_setup,
+                ),
+                kwargs=dict(
+                    skip_set_fem=True,
+                )
+            )
             t_main.start()
 
         # save history
@@ -1330,11 +1385,11 @@ class FEMOpt:
         t_save_history = Thread(target=save_history)
         t_save_history.start()
 
-
         # 終了を待つ
         self.client.gather(calc_futures)
         if not self.opt.is_cluster:  # 既存の fem を使っているならそれも待つ
-            t_main.join()
+            if t_main is not None:
+                t_main.join()
         self.status.set(OptimizationStatus.TERMINATED)
         end = time()
 
@@ -1357,6 +1412,8 @@ class FEMOpt:
         # terminate actors
         self.client.cancel(self.history._future, force=True)
         self.client.cancel(self.status._future, force=True)
+        for worker_status in self.worker_status_list:
+            self.client.cancel(worker_status._future, force=True)
         logger.info('Terminate actors.')
         sleep(1)
 
@@ -1393,4 +1450,3 @@ def start_monitor_server_forever(history, status, host='localhost', port=8080):
     monitor = Monitor(history, status)
     monitor.start_server(host, port)
     return 'Exit monitor server process gracefully'
-

{pyfemtet-0.3.3 → pyfemtet-0.3.5}/pyfemtet/opt/interface.py

@@ -9,7 +9,7 @@ import pandas as pd
 import psutil
 from pywintypes import com_error
 from pythoncom import CoInitialize, CoUninitialize
-from win32com.client import constants
+from win32com.client import constants, Dispatch
 from dask.distributed import get_worker
 from femtetutils import util
 
@@ -119,6 +119,7 @@ class FemtetInterface(FEMInterface):
             femprj_path=None,
             model_name=None,
             connect_method='auto',
+            strict_pid_specify=False,
             **kwargs
     ):
         """Initializes the FemtetInterface.
@@ -144,12 +145,13 @@ class FemtetInterface(FEMInterface):
         self.model_name = model_name
         self.connect_method = connect_method
 
-        # その他のメンバーの宣言や初期化初期化
+        # その他のメンバーの宣言や初期化
         self.Femtet = None
         self.quit_when_destruct = False
         self.connected_method = 'unconnected'
         self.parameters = None
         self.max_api_retry = 3
+        self.strict_pid_specify = strict_pid_specify
 
         # dask サブプロセスのときは femprj を更新し connect_method を new にする
         try:
@@ -198,9 +200,12 @@ class FemtetInterface(FEMInterface):
 
     def _connect_new_femtet(self):
         logger.info('└ Try to launch and connect new Femtet process.')
-        self.Femtet, _ = launch_and_dispatch_femtet()
+
+        self.Femtet, _ = launch_and_dispatch_femtet(strict_pid_specify=self.strict_pid_specify)
+
         self.connected_method = 'new'
 
+
     def _connect_existing_femtet(self, pid: int or None = None):
         logger.info('└ Try to connect existing Femtet process.')
         # 既存の Femtet を探して Dispatch する。
@@ -634,7 +639,7 @@ class FemtetWithNXInterface(FemtetInterface):
             femprj_path=femprj_path,
             model_name=model_name,
             connect_method=connect_method,
-            prt_path = os.path.basename(prt_path)  # upload_file でアップロードされたファイルへのパスになる
+            prt_path=os.path.basename(prt_path)  # upload_file でアップロードされたファイルへのパスになる
         )
 
 

{pyfemtet-0.3.3 → pyfemtet-0.3.5}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "pyfemtet"
-version = "0.3.3"  # ignored by versioning plugin
+version = "0.3.5"  # ignored by versioning plugin
 description = "Design parameter optimization using Femtet."
 authors = ["kazuma.naito <kazuma.naito@murata.com>"]
 readme = "README.md"

pyfemtet-0.3.3/pyfemtet/__init__.py

@@ -1 +0,0 @@
-__version__ = "0.3.3"