pyfemtet 0.4.21__py3-none-any.whl → 0.4.23__py3-none-any.whl

pyfemtet/__init__.py

@@ -1 +1 @@
-__version__ = "0.4.21"
+__version__ = "0.4.23"

pyfemtet/_test_util.py

@@ -103,8 +103,6 @@ def record_result(src: FEMOpt or str, py_path, suffix=''):
 
 
 def _get_obj_from_csv(csv_path, encoding='cp932'):
-    print('=====')
-    print(csv_path)
     df = pd.read_csv(csv_path, encoding=encoding, header=2)
     columns = df.columns
     with open(csv_path, mode='r', encoding=encoding, newline='\n') as f:

pyfemtet/message/messages.py

@@ -50,6 +50,8 @@ class Message:
                                         'Please free this file before exiting the program, '
                                         'otherwise history data will be lost.')
     OPTIMIZATION_FINISHED = _('Optimization finished. Results were saved in following:')
+    ERR_NO_BOUNDS = _('No bounds specified.')
+    CONFIRM_BEFORE_EXIT = _('The optimization is now complete. You can view the results on the monitor until you press Enter to exit the program.')
 
     # ===== pyfemtet.opt.interface =====
     ERR_RUN_JOURNAL_NOT_FOUND = _(r'"%UGII_BASE_DIR%\NXBIN\run_journal.exe" is not found. Make sure you have NX installed and the environment variable UGII_BASE_DIR is set.')
@@ -87,6 +89,7 @@ class Message:
     INFO_INFEASIBLE = _('The constraints were not satisfied for the following sets of variables:')
     ERR_FEM_FAILED_AND_CANNOT_CONTINUE = _('Current parameter set cannot update FEM and this optimization method cannot skip current parameter set. The optimization process will be terminated.')
     WARN_INTERRUPTED_IN_SCIPY = _('Optimization has been interrupted. Note that you cannot acquire the OptimizationResult in case of `trust-constr`, `TNC`, `SLSQP` or `COBYLA`.')
+    ERR_PARAMETER_CONSTRAINT_ONLY_BOTORCH = _('You can use parameter constraint only with BoTorchSampler.')
 
     # ===== pyfemtet.opt.visualization =====
     # control_femtet.py
@@ -120,6 +123,7 @@ class Message:
     ERR_NO_FEM_RESULT = _('No FEM result (yet).')
     ERR_NO_PREDICTION_MODEL = _('Prediction model is not calculated yet.')
     ERR_CANNOT_SELECT_SAME_PARAMETER = _('Cannot select same parameter')
+    LABEL_SWITCH_PREDICTION_MODEL_3D = _('3D graph (two or more parameters required)')
     # pm_graph_creator
     GRAPH_TITLE_PREDICTION_MODEL = _('Prediction Model of Objective')
     LEGEND_LABEL_PREDICTION_MODEL = _('prediction model')
@@ -128,6 +132,7 @@ class Message:
     PAGE_TITLE_PROGRESS = _('Progress')
     PAGE_TITLE_PREDICTION_MODEL = _('Prediction')
     PAGE_TITLE_WORKERS = _('Workers')
+    PAGE_TITLE_OPTUNA_VISUALIZATION = _('Details')
     # process monitor pages
     DEFAULT_STATUS_ALERT = _('Optimization status will be shown here.')
     LABEL_AUTO_UPDATE = _('Auto-update graph')
@@ -170,5 +175,14 @@ class Message:
     ERR_SAMPLE_CSV_NOT_FOUND = _('Sample csv is not found. Please consider to re-install pyfemtet by `py -m pip install pyfemtet -U --force-reinstall`')
     ERR_SAMPLE_FEMPRJ_NOT_FOUND = _('Sample femprj file is not found. Please consider to re-install pyfemtet by `py -m pip install pyfemtet -U --force-reinstall`')
     ERR_FEMPRJ_RESULT_NOT_FOUND = _('Sample femprj result folder is not found. Please consider to re-install pyfemtet by `py -m pip install pyfemtet -U --force-reinstall`')
-
+    # DETAIL VISUALIZATION PAGES
+    DETAIL_PAGE_TEXT_BEFORE_LOADING = _('Loading data...')
+    DETAIL_PAGE_HISTORY_HEADER = _('Plots of objectives versus trials')
+    DETAIL_PAGE_HISTORY_DESCRIPTION = _('The vertical axis is the objective, and the horizontal axis is the number of trials.')
+    DETAIL_PAGE_PARALLEL_COOR_HEADER = _('Parallel coordinate plots')
+    DETAIL_PAGE_PARALLEL_COOR_DESCRIPTION = _('The vertical axis is an objective or parameters, and one polyline indicates one result.')
+    DETAIL_PAGE_CONTOUR_HEADER = _('The heatmap of objectives')
+    DETAIL_PAGE_CONTOUR_DESCRIPTION = _('The axes are parameters, and the color shows objective value.')
+    DETAIL_PAGE_SLICE_HEADER = _('The response of an objective versus one parameter')
+    DETAIL_PAGE_SLICE_DESCRIPTION = _('The vertical axis is objective, and the horizontal axis is parameter.')
 

pyfemtet/opt/_femopt.py

@@ -1,10 +1,13 @@
 # built-in
-from typing import Optional, Any, Callable
+import inspect
+import warnings
+from typing import Optional, Any, Callable, List
 import os
 import datetime
 from time import time, sleep
 from threading import Thread
 import json
+from traceback import print_exception
 
 # 3rd-party
 import numpy as np
@@ -28,6 +31,29 @@ from pyfemtet.message import Msg, encoding
 from pyfemtet.opt.parameter import Parameter, Expression
 
 
+def add_worker(client, worker_name):
+    import sys
+    from subprocess import Popen, DEVNULL
+
+    current_n_workers = len(client.nthreads().keys())
+
+    Popen(
+        f'{sys.executable} -m dask worker '
+        f'{client.scheduler.address} '
+        f'--nthreads 1 '
+        f'--nworkers 1 '
+        f'--name {worker_name} '
+        f'--no-nanny',
+        shell=True,
+        stderr=DEVNULL,
+        stdout=DEVNULL,
+    )
+
+    # worker が増えるまで待つ
+    client.wait_for_workers(n_workers=current_n_workers + 1)
+
+
+
 class FEMOpt:
     """Class to control FEM interface and optimizer.
 
@@ -84,7 +110,6 @@ class FEMOpt:
         self.monitor_process_future = None
         self.monitor_server_kwargs = dict()
         self.monitor_process_worker_name = None
-        self._is_error_exit = False
 
     # multiprocess 時に pickle できないオブジェクト参照の削除
     def __getstate__(self):
@@ -282,6 +307,47 @@ class FEMOpt:
 
         self.opt.constraints[name] = Constraint(fun, name, lower_bound, upper_bound, strict, args, kwargs)
 
+    def add_parameter_constraint(
+            self,
+            fun: Callable[[List[float], Any], float],
+            name: Optional[str] = None,
+            lower_bound: Optional[float] = None,
+            upper_bound: Optional[float] = None,
+    ):
+        """Add constraint in case of parameter-only.
+
+        Args:
+            fun (Callable[[List[float], Any], float]): Function to constraint. The name of arguments must be one of ones of parameter or variables.
+            name (Optional[str], optional): Name of constraint. Defaults to None and then name is set to 'cns_{i}'.
+            lower_bound (Optional[float], optional): Lower bound of return value. Defaults to None.
+            upper_bound (Optional[float], optional): Upper bound of return value. Defaults to None.
+        """
+
+        # candidate default name
+        if name is None:
+            prefix = Constraint.default_name
+            i = 0
+            while True:
+                candidate = f'{prefix}_{str(int(i))}'
+                is_existing = candidate in list(self.opt.constraints.keys())
+                if not is_existing:
+                    break
+                else:
+                    i += 1
+            name = candidate
+
+        # assert at least 1 bound exist
+        assert lower_bound is not None or upper_bound is not None, Msg.ERR_NO_BOUNDS
+
+        from pyfemtet.opt._femopt_core import ParameterConstraint
+        self.opt.constraints[name] = ParameterConstraint(fun, name, lower_bound, upper_bound, self.opt)
+        if hasattr(self.opt, 'add_parameter_constraints'):
+            prm_args = [p.name for p in inspect.signature(fun).parameters.values()]
+            if lower_bound is not None:
+                self.opt.add_parameter_constraints(lambda *args, **kwargs: fun(*args, **kwargs) - lower_bound, prm_args=prm_args)
+            if upper_bound is not None:
+                self.opt.add_parameter_constraints(lambda *args, **kwargs: upper_bound - fun(*args, **kwargs), prm_args=prm_args)
+
     def get_parameter(self, format='dict'):
         raise DeprecationWarning('FEMOpt.get_parameter() was deprecated. Use Femopt.opt.get_parameter() instead.')
 
@@ -312,6 +378,7 @@ class FEMOpt:
             n_parallel=1,
             timeout=None,
             wait_setup=True,
+            confirm_before_exit=True,
     ):
         """Runs the main optimization process.
 
@@ -320,6 +387,7 @@ class FEMOpt:
             n_parallel (int, optional): The number of parallel processes. Defaults to 1.
             timeout (float or None, optional): The maximum amount of time in seconds that each trial can run. Defaults to None.
             wait_setup (bool, optional): Wait for all workers launching FEM system. Defaults to True.
+            confirm_before_exit (bool, optional): Insert stop before exit to continue to show process monitor.
 
         Tip:
             If set_monitor_host() is not executed, a local server for monitoring will be started at localhost:8080.
@@ -398,16 +466,7 @@ class FEMOpt:
             # monitor worker の設定
             logger.info('Launching monitor server. This may take a few seconds.')
             self.monitor_process_worker_name = datetime.datetime.now().strftime("Monitor%Y%m%d%H%M%S")
-            current_n_workers = len(self.client.nthreads().keys())
-            from subprocess import Popen
-            import sys
-            Popen(
-                f'{sys.executable} -m dask worker {self.client.scheduler.address} --nthreads 1 --nworkers 1 --name {self.monitor_process_worker_name} --no-nanny',
-                shell=True
-            )
-
-            # monitor 用 worker が増えるまで待つ
-            self.client.wait_for_workers(n_workers=current_n_workers + 1)
+            add_worker(self.client, self.monitor_process_worker_name)
 
         else:
             # ローカルクラスターを構築
@@ -425,200 +484,175 @@ class FEMOpt:
             self.monitor_process_worker_name = worker_addresses[0]
             worker_addresses[0] = 'Main'
 
-        # Femtet 特有の処理
-        metadata = None
-        if isinstance(self.fem, FemtetInterface):
-            # 結果 csv に記載する femprj に関する情報
-            metadata = json.dumps(
-                dict(
-                    femprj_path=self.fem.original_femprj_path,
-                    model_name=self.fem.model_name
+        with self.client.cluster as _cluster, self.client as _client:
+
+            # Femtet 特有の処理
+            metadata = None
+            if isinstance(self.fem, FemtetInterface):
+                # 結果 csv に記載する femprj に関する情報
+                metadata = json.dumps(
+                    dict(
+                        femprj_path=self.fem.original_femprj_path,
+                        model_name=self.fem.model_name
+                    )
                 )
+                # Femtet の parametric 設定を目的関数に用いるかどうか
+                if self.fem.parametric_output_indexes_use_as_objective is not None:
+                    from pyfemtet.opt.interface._femtet_parametric import add_parametric_results_as_objectives
+                    indexes = list(self.fem.parametric_output_indexes_use_as_objective.keys())
+                    directions = list(self.fem.parametric_output_indexes_use_as_objective.values())
+                    add_parametric_results_as_objectives(
+                        self,
+                        indexes,
+                        directions,
+                    )
+
+            # actor の設定
+            self.status = OptimizationStatus(_client)
+            self.worker_status_list = [OptimizationStatus(_client, name) for name in worker_addresses]  # tqdm 検討
+            self.status.set(OptimizationStatus.SETTING_UP)
+            self.history = History(
+                self.history_path,
+                self.opt.variables.get_parameter_names(),
+                list(self.opt.objectives.keys()),
+                list(self.opt.constraints.keys()),
+                _client,
+                metadata,
             )
-            # Femtet の parametric 設定を目的関数に用いるかどうか
-            if self.fem.parametric_output_indexes_use_as_objective is not None:
-                from pyfemtet.opt.interface._femtet_parametric import add_parametric_results_as_objectives
-                indexes = list(self.fem.parametric_output_indexes_use_as_objective.keys())
-                directions = list(self.fem.parametric_output_indexes_use_as_objective.values())
-                add_parametric_results_as_objectives(
-                    self,
-                    indexes,
-                    directions,
-                )
-
-        # actor の設定
-        self.status = OptimizationStatus(self.client)
-        self.worker_status_list = [OptimizationStatus(self.client, name) for name in worker_addresses]  # tqdm 検討
-        self.status.set(OptimizationStatus.SETTING_UP)
-        self.history = History(
-            self.history_path,
-            self.opt.variables.get_parameter_names(),
-            list(self.opt.objectives.keys()),
-            list(self.opt.constraints.keys()),
-            self.client,
-            metadata,
-        )
-
-        # launch monitor
-        self.monitor_process_future = self.client.submit(
-            # func
-            _start_monitor_server,
-            # args
-            self.history,
-            self.status,
-            worker_addresses,
-            self.worker_status_list,
-            # kwargs
-            **self.monitor_server_kwargs,
-            # kwargs of submit
-            workers=self.monitor_process_worker_name,
-            allow_other_workers=False
-        )
-
-        # fem
-        self.fem._setup_before_parallel(self.client)
-
-        # opt
-        self.opt.fem_class = type(self.fem)
-        self.opt.fem_kwargs = self.fem.kwargs
-        self.opt.entire_status = self.status
-        self.opt.history = self.history
-        self.opt._setup_before_parallel()
-
-        # クラスターでの計算開始
-        self.status.set(OptimizationStatus.LAUNCHING_FEM)
-        start = time()
-        calc_futures = self.client.map(
-            self.opt._run,
-            subprocess_indices,
-            [self.worker_status_list] * len(subprocess_indices),
-            [wait_setup] * len(subprocess_indices),
-            workers=worker_addresses,
-            allow_other_workers=False,
-        )
 
-        t_main = None
-        if not self.opt.is_cluster:
-            # ローカルプロセスでの計算(opt._main 相当の処理)
-            subprocess_idx = 0
-
-            # set_fem
-            self.opt.fem = self.fem
-            self.opt._reconstruct_fem(skip_reconstruct=True)
-
-            t_main = Thread(
-                target=self.opt._run,
-                args=(
-                    subprocess_idx,
-                    self.worker_status_list,
-                    wait_setup,
-                ),
-                kwargs=dict(
-                    skip_set_fem=True,
-                )
+            # launch monitor
+            self.monitor_process_future = _client.submit(
+                # func
+                _start_monitor_server,
+                # args
+                self.history,
+                self.status,
+                worker_addresses,
+                self.worker_status_list,
+                # kwargs
+                **self.monitor_server_kwargs,
+                # kwargs of submit
+                workers=self.monitor_process_worker_name,
+                allow_other_workers=False
             )
-            t_main.start()
-
-        # save history
-        def save_history():
-            while True:
-                sleep(2)
-                try:
-                    self.history.save()
-                except PermissionError:
-                    logger.warning(Msg.WARN_HISTORY_CSV_NOT_ACCESSIBLE)
-                if self.status.get() >= OptimizationStatus.TERMINATED:
-                    break
-
-        t_save_history = Thread(target=save_history)
-        t_save_history.start()
-
-        # 終了を待つ
-        local_opt_crashed = False
-        opt_crashed_list = self.client.gather(calc_futures)
-        if not self.opt.is_cluster:  # 既存の fem を使っているならそれも待つ
-            if t_main is not None:
-                t_main.join()
-                local_opt_crashed = self.opt._is_error_exit
-        opt_crashed_list.append(local_opt_crashed)
-        self.status.set(OptimizationStatus.TERMINATED)
-        end = time()
-
-        # 一応
-        t_save_history.join()
-
-        # logger.info(f'計算が終了しました. 実行時間は {int(end - start)} 秒でした。ウィンドウを閉じると終了します.')
-        # logger.info(f'結果は{self.history.path}を確認してください.')
-        logger.info(Msg.OPTIMIZATION_FINISHED)
-        logger.info(self.history.path)
 
-        # ひとつでも crashed ならばフラグを立てる
-        if any(opt_crashed_list):
-            self._is_error_exit = True
-        
-        return self.history.local_data
-
-
-    def terminate_all(self):
-        """Try to terminate all launched processes.
-
-        If distributed computing, Scheduler and Workers will NOT be terminated.
-
-        """
-
-        # monitor が terminated 状態で少なくとも一度更新されなければ running のまま固まる
-        sleep(1)
-
-        # terminate monitor process
-        self.status.set(OptimizationStatus.TERMINATE_ALL)
-        logger.info(self.monitor_process_future.result())
-        sleep(1)
-
-        # terminate actors
-        self.client.cancel(self.history._future, force=True)
-        self.client.cancel(self.status._future, force=True)
-        for worker_status in self.worker_status_list:
-            self.client.cancel(worker_status._future, force=True)
-        logger.info('Terminate actors.')
-        sleep(1)
-
-        # terminate monitor worker
-        n_workers = len(self.client.nthreads())
-
-        found_worker_dict = self.client.retire_workers(
-            names=[self.monitor_process_worker_name],  # name
-            close_workers=True,
-            remove=True,
-        )
-
-        if len(found_worker_dict) == 0:
-            found_worker_dict = self.client.retire_workers(
-                workers=[self.monitor_process_worker_name],  # address
-                close_workers=True,
-                remove=True,
+            # fem
+            self.fem._setup_before_parallel(_client)
+
+            # opt
+            self.opt.fem_class = type(self.fem)
+            self.opt.fem_kwargs = self.fem.kwargs
+            self.opt.entire_status = self.status
+            self.opt.history = self.history
+            self.opt._setup_before_parallel()
+
+            # クラスターでの計算開始
+            self.status.set(OptimizationStatus.LAUNCHING_FEM)
+            start = time()
+            calc_futures = _client.map(
+                self.opt._run,
+                subprocess_indices,
+                [self.worker_status_list] * len(subprocess_indices),
+                [wait_setup] * len(subprocess_indices),
+                workers=worker_addresses,
+                allow_other_workers=False,
             )
 
-        if len(found_worker_dict) > 0:
-            while n_workers == len(self.client.nthreads()):
-                sleep(1)
-            logger.info('Terminate monitor processes worker.')
-            sleep(1)
-        else:
-            logger.warn('Monitor process worker not found.')
-
-        # close FEM (if specified to quit when deconstruct)
-        del self.fem
-        logger.info('Terminate FEM.')
-        sleep(1)
-
-        # close scheduler, other workers(, cluster)
-        self.client.shutdown()
-        logger.info('Terminate all relative processes.')
-        sleep(3)
-
-        # if optimization was crashed, raise Exception
-        if self._is_error_exit:
-            raise RuntimeError('At least 1 of optimization processes have been crashed. See console log.')
+            t_main = None
+            if not self.opt.is_cluster:
+                # ローカルプロセスでの計算(opt._main 相当の処理)
+                subprocess_idx = 0
+
+                # set_fem
+                self.opt.fem = self.fem
+                self.opt._reconstruct_fem(skip_reconstruct=True)
+
+                t_main = Thread(
+                    target=self.opt._run,
+                    args=(
+                        subprocess_idx,
+                        self.worker_status_list,
+                        wait_setup,
+                    ),
+                    kwargs=dict(
+                        skip_set_fem=True,
+                    )
+                )
+                t_main.start()
+
+            # save history
+            def save_history():
+                while True:
+                    sleep(2)
+                    try:
+                        self.history.save()
+                    except PermissionError:
+                        logger.warning(Msg.WARN_HISTORY_CSV_NOT_ACCESSIBLE)
+                    if self.status.get() >= OptimizationStatus.TERMINATED:
+                        break
+
+            t_save_history = Thread(target=save_history)
+            t_save_history.start()
+
+            # ===== 終了 =====
+
+            # クラスターの Unexpected Exception のリストを取得
+            opt_exceptions: list[Exception or None] = _client.gather(calc_futures)  # gather() で終了待ちも兼ねる
+
+            # ローカルの opt で計算している場合、その Exception も取得
+            local_opt_exception: Exception or None = None
+            if not self.opt.is_cluster:
+                if t_main is not None:
+                    t_main.join()  # 終了待ち
+                    local_opt_exception = self.opt._exception  # Exception を取得
+            opt_exceptions.append(local_opt_exception)
+
+            # 終了
+            self.status.set(OptimizationStatus.TERMINATED)
+            end = time()
+
+            # 一応
+            t_save_history.join()
+
+            # 結果通知
+            logger.info(Msg.OPTIMIZATION_FINISHED)
+            logger.info(self.history.path)
+
+            # monitor worker を終了する準備
+            # 実際の終了は monitor worker の終了時
+            self.status.set(OptimizationStatus.TERMINATE_ALL)
+            logger.info(self.monitor_process_future.result())
+            sleep(1)  # monitor が terminated 状態で少なくとも一度更新されなければ running のまま固まる
+
+            # 全ての Exception を再表示
+            for i, opt_exception in enumerate(opt_exceptions):
+                if opt_exception is not None:
+                    print()
+                    print(f'===== unexpected exception raised on worker {i} =====')
+                    print_exception(opt_exception)
+                    print()
+
+            # monitor worker を残してユーザーが結果を確認できるようにする
+            if confirm_before_exit:
+                print()
+                print('='*len(Msg.CONFIRM_BEFORE_EXIT))
+                print(Msg.CONFIRM_BEFORE_EXIT)
+                print('='*len(Msg.CONFIRM_BEFORE_EXIT))
+                input()
+
+            return self.history.get_df()  # with 文を抜けると actor は消えるが .copy() はこの段階では不要
+
+    @staticmethod
+    def terminate_all():
+        warnings.warn(
+            "terminate_all() is deprecated and will be removed in a future version. "
+            "In current and later versions, the equivalent of terminate_all() will be executed when optimize() finishes. "
+            "Therefore, you can simply remove terminate_all() from your code. "
+            "If you want to stop program before terminating monitor process, "
+            "use ``confirm_before_exit`` argument like ``FEMOpt.optimize(confirm_before_exit=True)``",
+            DeprecationWarning,
+            stacklevel=2
+        )
 
 
 def _start_monitor_server(