pyfemtet 0.4.20__py3-none-any.whl → 0.4.23__py3-none-any.whl

pyfemtet/opt/_femopt.py

@@ -1,9 +1,13 @@
 # built-in
+import inspect
+import warnings
+from typing import Optional, Any, Callable, List
 import os
 import datetime
 from time import time, sleep
 from threading import Thread
 import json
+from traceback import print_exception
 
 # 3rd-party
 import numpy as np
@@ -24,6 +28,30 @@ from pyfemtet.opt._femopt_core import (
     logger,
 )
 from pyfemtet.message import Msg, encoding
+from pyfemtet.opt.parameter import Parameter, Expression
+
+
+def add_worker(client, worker_name):
+    import sys
+    from subprocess import Popen, DEVNULL
+
+    current_n_workers = len(client.nthreads().keys())
+
+    Popen(
+        f'{sys.executable} -m dask worker '
+        f'{client.scheduler.address} '
+        f'--nthreads 1 '
+        f'--nworkers 1 '
+        f'--name {worker_name} '
+        f'--no-nanny',
+        shell=True,
+        stderr=DEVNULL,
+        stdout=DEVNULL,
+    )
+
+    # worker が増えるまで待つ
+    client.wait_for_workers(n_workers=current_n_workers + 1)
+
 
 
 class FEMOpt:
@@ -82,7 +110,6 @@ class FEMOpt:
         self.monitor_process_future = None
         self.monitor_server_kwargs = dict()
         self.monitor_process_worker_name = None
-        self._is_error_exit = False
 
     # multiprocess 時に pickle できないオブジェクト参照の削除
     def __getstate__(self):
@@ -109,7 +136,8 @@ class FEMOpt:
             lower_bound: float or None = None,
             upper_bound: float or None = None,
             step: float or None = None,
-            memo: str = ''
+            properties: Optional[dict] = None,
+            pass_to_fem: Optional[bool] = True,
     ):
         """Adds a parameter to the optimization problem.
 
@@ -119,34 +147,63 @@ class FEMOpt:
             lower_bound (float or None, optional): The lower bound of the parameter. Defaults to None. However, this argument is required for some algorithms.
             upper_bound (float or None, optional): The upper bound of the parameter. Defaults to None. However, this argument is required for some algorithms.
             step (float or None, optional): The step of parameter. Defaults to None.
-            memo (str, optional): Additional information about the parameter. Defaults to ''.
+            properties (dict, optional): Additional information about the parameter. Defaults to None.
+            pass_to_fem (bool, optional): If this variable is used directly in FEM model update or not. If False, this parameter can be just used as inpt of expressions. Defaults to True.
         Raises:
             ValueError: If initial_value is not specified and the value for the given name is also not specified.
 
         """
 
         _check_bound(lower_bound, upper_bound, name)
-        value = self.fem.check_param_value(name)
-        if initial_value is None:
-            if value is not None:
-                initial_value = value
-            else:
-                raise ValueError('initial_value を指定してください.')
 
-        d = {
-            'name': name,
-            'value': float(initial_value),
-            'lb': float(lower_bound) if lower_bound is not None else None,
-            'ub': float(upper_bound) if upper_bound is not None else None,
-            'step': float(step) if step is not None else None,
-            'memo': memo,
-        }
-        pdf = pd.DataFrame(d, index=[0], dtype=object)
-
-        if len(self.opt.parameters) == 0:
-            self.opt.parameters = pdf
+        if pass_to_fem:
+            value = self.fem.check_param_value(name)
+            if initial_value is None:
+                if value is not None:
+                    initial_value = value
+                else:
+                    raise ValueError('initial_value を指定してください.')
         else:
-            self.opt.parameters = pd.concat([self.opt.parameters, pdf], ignore_index=True)
+            if initial_value is None:
+                raise ValueError('initial_value を指定してください.')
+
+        prm = Parameter(
+            name=name,
+            value=float(initial_value),
+            lower_bound=float(lower_bound) if lower_bound is not None else None,
+            upper_bound=float(upper_bound) if upper_bound is not None else None,
+            step=float(step) if step is not None else None,
+            pass_to_fem=pass_to_fem,
+            properties=properties,
+        )
+        self.opt.variables.add_parameter(prm)
+
+    def add_expression(
+            self,
+            name: str,
+            fun: Callable[[Any], float],
+            properties=property,
+            kwargs: Optional[dict] = None,
+            pass_to_fem=True,
+    ):
+        """Add expression to the optimization problem.
+
+        Args:
+            name (str): The name of the variable.
+            fun (Callable[[Any], float]): An expression function. The arguments that you want to use as input variables must be the same with ``name`` of Variable objects added by ``add_parameter()`` or ``add_expression()``. If you use other objects as argument of the function, you must specify ``kwargs``.
+            properties ([type], optional): Property names and their values of the variable. Defaults to property.
+            kwargs (Optional[dict], optional): Remaining arguments of ``fun``. Defaults to None.
+            pass_to_fem (bool, optional): If this variable is used directly in FEM model update or not. If False, this variable can be just used as inpt of other expressions. Defaults to True.
+        """
+        exp = Expression(
+            name=name,
+            value=None,
+            properties=properties,
+            fun=fun,
+            kwargs=kwargs if kwargs else {},
+            pass_to_fem=pass_to_fem,
+        )
+        self.opt.variables.add_expression(exp)
 
     def add_objective(
             self,
@@ -250,20 +307,49 @@ class FEMOpt:
 
         self.opt.constraints[name] = Constraint(fun, name, lower_bound, upper_bound, strict, args, kwargs)
 
-    def get_parameter(self, format='dict') -> pd.DataFrame or dict or np.ndarray:
-        """Returns the parameter in a specified format.
+    def add_parameter_constraint(
+            self,
+            fun: Callable[[List[float], Any], float],
+            name: Optional[str] = None,
+            lower_bound: Optional[float] = None,
+            upper_bound: Optional[float] = None,
+    ):
+        """Add constraint in case of parameter-only.
 
         Args:
-            format (str, optional): The desired output format. Defaults to 'dict'. Valid formats are 'values', 'df' and 'dict'.
+            fun (Callable[[List[float], Any], float]): Function to constraint. The name of arguments must be one of ones of parameter or variables.
+            name (Optional[str], optional): Name of constraint. Defaults to None and then name is set to 'cns_{i}'.
+            lower_bound (Optional[float], optional): Lower bound of return value. Defaults to None.
+            upper_bound (Optional[float], optional): Upper bound of return value. Defaults to None.
+        """
 
-        Returns:
-            pd.DataFrame or dict or np.ndarray: The parameter data converted into the specified format.
+        # candidate default name
+        if name is None:
+            prefix = Constraint.default_name
+            i = 0
+            while True:
+                candidate = f'{prefix}_{str(int(i))}'
+                is_existing = candidate in list(self.opt.constraints.keys())
+                if not is_existing:
+                    break
+                else:
+                    i += 1
+            name = candidate
 
-        Raises:
-            ValueError: If an invalid format is provided.
+        # assert at least 1 bound exist
+        assert lower_bound is not None or upper_bound is not None, Msg.ERR_NO_BOUNDS
 
-        """
-        return self.opt.get_parameter(format)
+        from pyfemtet.opt._femopt_core import ParameterConstraint
+        self.opt.constraints[name] = ParameterConstraint(fun, name, lower_bound, upper_bound, self.opt)
+        if hasattr(self.opt, 'add_parameter_constraints'):
+            prm_args = [p.name for p in inspect.signature(fun).parameters.values()]
+            if lower_bound is not None:
+                self.opt.add_parameter_constraints(lambda *args, **kwargs: fun(*args, **kwargs) - lower_bound, prm_args=prm_args)
+            if upper_bound is not None:
+                self.opt.add_parameter_constraints(lambda *args, **kwargs: upper_bound - fun(*args, **kwargs), prm_args=prm_args)
+
+    def get_parameter(self, format='dict'):
+        raise DeprecationWarning('FEMOpt.get_parameter() was deprecated. Use Femopt.opt.get_parameter() instead.')
 
     def set_monitor_host(self, host=None, port=None):
         """Sets up the monitor server with the specified host and port.
@@ -292,6 +378,7 @@ class FEMOpt:
             n_parallel=1,
             timeout=None,
             wait_setup=True,
+            confirm_before_exit=True,
     ):
         """Runs the main optimization process.
 
@@ -300,6 +387,7 @@ class FEMOpt:
             n_parallel (int, optional): The number of parallel processes. Defaults to 1.
             timeout (float or None, optional): The maximum amount of time in seconds that each trial can run. Defaults to None.
             wait_setup (bool, optional): Wait for all workers launching FEM system. Defaults to True.
+            confirm_before_exit (bool, optional): Insert stop before exit to continue to show process monitor.
 
         Tip:
             If set_monitor_host() is not executed, a local server for monitoring will be started at localhost:8080.
@@ -338,8 +426,9 @@ class FEMOpt:
             self.opt.method_checker.check_seed()
 
         is_incomplete_bounds = False
-        for _, row in self.opt.parameters.iterrows():
-            lb, ub = row['lb'], row['ub']
+        prm: Parameter = None
+        for prm in self.opt.variables.parameters.values():
+            lb, ub = prm.lower_bound, prm.upper_bound
             is_incomplete_bounds = is_incomplete_bounds + (lb is None) + (ub is None)
         if is_incomplete_bounds:
             self.opt.method_checker.check_incomplete_bounds()
@@ -348,6 +437,10 @@ class FEMOpt:
         self.opt.n_trials = n_trials
         self.opt.timeout = timeout
 
+        # resolve expression dependencies
+        self.opt.variables.resolve()
+        self.opt.variables.evaluate()
+
         # クラスターの設定
         self.opt.is_cluster = self.scheduler_address is not None
         if self.opt.is_cluster:
@@ -373,16 +466,7 @@ class FEMOpt:
             # monitor worker の設定
             logger.info('Launching monitor server. This may take a few seconds.')
             self.monitor_process_worker_name = datetime.datetime.now().strftime("Monitor%Y%m%d%H%M%S")
-            current_n_workers = len(self.client.nthreads().keys())
-            from subprocess import Popen
-            import sys
-            Popen(
-                f'{sys.executable} -m dask worker {self.client.scheduler.address} --nthreads 1 --nworkers 1 --name {self.monitor_process_worker_name} --no-nanny',
-                shell=True
-            )
-
-            # monitor 用 worker が増えるまで待つ
-            self.client.wait_for_workers(n_workers=current_n_workers + 1)
+            add_worker(self.client, self.monitor_process_worker_name)
 
         else:
             # ローカルクラスターを構築
@@ -400,200 +484,175 @@ class FEMOpt:
             self.monitor_process_worker_name = worker_addresses[0]
             worker_addresses[0] = 'Main'
 
-        # Femtet 特有の処理
-        metadata = None
-        if isinstance(self.fem, FemtetInterface):
-            # 結果 csv に記載する femprj に関する情報
-            metadata = json.dumps(
-                dict(
-                    femprj_path=self.fem.original_femprj_path,
-                    model_name=self.fem.model_name
+        with self.client.cluster as _cluster, self.client as _client:
+
+            # Femtet 特有の処理
+            metadata = None
+            if isinstance(self.fem, FemtetInterface):
+                # 結果 csv に記載する femprj に関する情報
+                metadata = json.dumps(
+                    dict(
+                        femprj_path=self.fem.original_femprj_path,
+                        model_name=self.fem.model_name
+                    )
                 )
+                # Femtet の parametric 設定を目的関数に用いるかどうか
+                if self.fem.parametric_output_indexes_use_as_objective is not None:
+                    from pyfemtet.opt.interface._femtet_parametric import add_parametric_results_as_objectives
+                    indexes = list(self.fem.parametric_output_indexes_use_as_objective.keys())
+                    directions = list(self.fem.parametric_output_indexes_use_as_objective.values())
+                    add_parametric_results_as_objectives(
+                        self,
+                        indexes,
+                        directions,
+                    )
+
+            # actor の設定
+            self.status = OptimizationStatus(_client)
+            self.worker_status_list = [OptimizationStatus(_client, name) for name in worker_addresses]  # tqdm 検討
+            self.status.set(OptimizationStatus.SETTING_UP)
+            self.history = History(
+                self.history_path,
+                self.opt.variables.get_parameter_names(),
+                list(self.opt.objectives.keys()),
+                list(self.opt.constraints.keys()),
+                _client,
+                metadata,
             )
-            # Femtet の parametric 設定を目的関数に用いるかどうか
-            if self.fem.parametric_output_indexes_use_as_objective is not None:
-                from pyfemtet.opt.interface._femtet_parametric import add_parametric_results_as_objectives
-                indexes = list(self.fem.parametric_output_indexes_use_as_objective.keys())
-                directions = list(self.fem.parametric_output_indexes_use_as_objective.values())
-                add_parametric_results_as_objectives(
-                    self,
-                    indexes,
-                    directions,
-                )
-
-        # actor の設定
-        self.status = OptimizationStatus(self.client)
-        self.worker_status_list = [OptimizationStatus(self.client, name) for name in worker_addresses]  # tqdm 検討
-        self.status.set(OptimizationStatus.SETTING_UP)
-        self.history = History(
-            self.history_path,
-            self.opt.parameters['name'].to_list(),
-            list(self.opt.objectives.keys()),
-            list(self.opt.constraints.keys()),
-            self.client,
-            metadata,
-        )
 
-        # launch monitor
-        self.monitor_process_future = self.client.submit(
-            # func
-            _start_monitor_server,
-            # args
-            self.history,
-            self.status,
-            worker_addresses,
-            self.worker_status_list,
-            # kwargs
-            **self.monitor_server_kwargs,
-            # kwargs of submit
-            workers=self.monitor_process_worker_name,
-            allow_other_workers=False
-        )
-
-        # fem
-        self.fem._setup_before_parallel(self.client)
-
-        # opt
-        self.opt.fem_class = type(self.fem)
-        self.opt.fem_kwargs = self.fem.kwargs
-        self.opt.entire_status = self.status
-        self.opt.history = self.history
-        self.opt._setup_before_parallel()
-
-        # クラスターでの計算開始
-        self.status.set(OptimizationStatus.LAUNCHING_FEM)
-        start = time()
-        calc_futures = self.client.map(
-            self.opt._run,
-            subprocess_indices,
-            [self.worker_status_list] * len(subprocess_indices),
-            [wait_setup] * len(subprocess_indices),
-            workers=worker_addresses,
-            allow_other_workers=False,
-        )
-
-        t_main = None
-        if not self.opt.is_cluster:
-            # ローカルプロセスでの計算(opt._main 相当の処理)
-            subprocess_idx = 0
-
-            # set_fem
-            self.opt.fem = self.fem
-            self.opt._reconstruct_fem(skip_reconstruct=True)
-
-            t_main = Thread(
-                target=self.opt._run,
-                args=(
-                    subprocess_idx,
-                    self.worker_status_list,
-                    wait_setup,
-                ),
-                kwargs=dict(
-                    skip_set_fem=True,
-                )
+            # launch monitor
+            self.monitor_process_future = _client.submit(
+                # func
+                _start_monitor_server,
+                # args
+                self.history,
+                self.status,
+                worker_addresses,
+                self.worker_status_list,
+                # kwargs
+                **self.monitor_server_kwargs,
+                # kwargs of submit
+                workers=self.monitor_process_worker_name,
+                allow_other_workers=False
             )
-            t_main.start()
-
-        # save history
-        def save_history():
-            while True:
-                sleep(2)
-                try:
-                    self.history.save()
-                except PermissionError:
-                    logger.warning(Msg.WARN_HISTORY_CSV_NOT_ACCESSIBLE)
-                if self.status.get() >= OptimizationStatus.TERMINATED:
-                    break
-
-        t_save_history = Thread(target=save_history)
-        t_save_history.start()
 
-        # 終了を待つ
-        local_opt_crashed = False
-        opt_crashed_list = self.client.gather(calc_futures)
-        if not self.opt.is_cluster:  # 既存の fem を使っているならそれも待つ
-            if t_main is not None:
-                t_main.join()
-                local_opt_crashed = self.opt._is_error_exit
-        opt_crashed_list.append(local_opt_crashed)
-        self.status.set(OptimizationStatus.TERMINATED)
-        end = time()
-
-        # 一応
-        t_save_history.join()
-
-        # logger.info(f'計算が終了しました. 実行時間は {int(end - start)} 秒でした。ウィンドウを閉じると終了します.')
-        # logger.info(f'結果は{self.history.path}を確認してください.')
-        logger.info(Msg.OPTIMIZATION_FINISHED)
-        logger.info(self.history.path)
-
-        # ひとつでも crashed ならばフラグを立てる
-        if any(opt_crashed_list):
-            self._is_error_exit = True
-        
-        return self.history.local_data
-
-
-    def terminate_all(self):
-        """Try to terminate all launched processes.
-
-        If distributed computing, Scheduler and Workers will NOT be terminated.
-
-        """
-
-        # monitor が terminated 状態で少なくとも一度更新されなければ running のまま固まる
-        sleep(1)
-
-        # terminate monitor process
-        self.status.set(OptimizationStatus.TERMINATE_ALL)
-        logger.info(self.monitor_process_future.result())
-        sleep(1)
-
-        # terminate actors
-        self.client.cancel(self.history._future, force=True)
-        self.client.cancel(self.status._future, force=True)
-        for worker_status in self.worker_status_list:
-            self.client.cancel(worker_status._future, force=True)
-        logger.info('Terminate actors.')
-        sleep(1)
-
-        # terminate monitor worker
-        n_workers = len(self.client.nthreads())
-
-        found_worker_dict = self.client.retire_workers(
-            names=[self.monitor_process_worker_name],  # name
-            close_workers=True,
-            remove=True,
-        )
-
-        if len(found_worker_dict) == 0:
-            found_worker_dict = self.client.retire_workers(
-                workers=[self.monitor_process_worker_name],  # address
-                close_workers=True,
-                remove=True,
+            # fem
+            self.fem._setup_before_parallel(_client)
+
+            # opt
+            self.opt.fem_class = type(self.fem)
+            self.opt.fem_kwargs = self.fem.kwargs
+            self.opt.entire_status = self.status
+            self.opt.history = self.history
+            self.opt._setup_before_parallel()
+
+            # クラスターでの計算開始
+            self.status.set(OptimizationStatus.LAUNCHING_FEM)
+            start = time()
+            calc_futures = _client.map(
+                self.opt._run,
+                subprocess_indices,
+                [self.worker_status_list] * len(subprocess_indices),
+                [wait_setup] * len(subprocess_indices),
+                workers=worker_addresses,
+                allow_other_workers=False,
             )
 
-        if len(found_worker_dict) > 0:
-            while n_workers == len(self.client.nthreads()):
-                sleep(1)
-            logger.info('Terminate monitor processes worker.')
-            sleep(1)
-        else:
-            logger.warn('Monitor process worker not found.')
-
-        # close FEM (if specified to quit when deconstruct)
-        del self.fem
-        logger.info('Terminate FEM.')
-        sleep(1)
-
-        # close scheduler, other workers(, cluster)
-        self.client.shutdown()
-        logger.info('Terminate all relative processes.')
-        sleep(3)
-
-        # if optimization was crashed, raise Exception
-        if self._is_error_exit:
-            raise RuntimeError('At least 1 of optimization processes have been crashed. See console log.')
+            t_main = None
+            if not self.opt.is_cluster:
+                # ローカルプロセスでの計算(opt._main 相当の処理)
+                subprocess_idx = 0
+
+                # set_fem
+                self.opt.fem = self.fem
+                self.opt._reconstruct_fem(skip_reconstruct=True)
+
+                t_main = Thread(
+                    target=self.opt._run,
+                    args=(
+                        subprocess_idx,
+                        self.worker_status_list,
+                        wait_setup,
+                    ),
+                    kwargs=dict(
+                        skip_set_fem=True,
+                    )
+                )
+                t_main.start()
+
+            # save history
+            def save_history():
+                while True:
+                    sleep(2)
+                    try:
+                        self.history.save()
+                    except PermissionError:
+                        logger.warning(Msg.WARN_HISTORY_CSV_NOT_ACCESSIBLE)
+                    if self.status.get() >= OptimizationStatus.TERMINATED:
+                        break
+
+            t_save_history = Thread(target=save_history)
+            t_save_history.start()
+
+            # ===== 終了 =====
+
+            # クラスターの Unexpected Exception のリストを取得
+            opt_exceptions: list[Exception or None] = _client.gather(calc_futures)  # gather() で終了待ちも兼ねる
+
+            # ローカルの opt で計算している場合、その Exception も取得
+            local_opt_exception: Exception or None = None
+            if not self.opt.is_cluster:
+                if t_main is not None:
+                    t_main.join()  # 終了待ち
+                    local_opt_exception = self.opt._exception  # Exception を取得
+            opt_exceptions.append(local_opt_exception)
+
+            # 終了
+            self.status.set(OptimizationStatus.TERMINATED)
+            end = time()
+
+            # 一応
+            t_save_history.join()
+
+            # 結果通知
+            logger.info(Msg.OPTIMIZATION_FINISHED)
+            logger.info(self.history.path)
+
+            # monitor worker を終了する準備
+            # 実際の終了は monitor worker の終了時
+            self.status.set(OptimizationStatus.TERMINATE_ALL)
+            logger.info(self.monitor_process_future.result())
+            sleep(1)  # monitor が terminated 状態で少なくとも一度更新されなければ running のまま固まる
+
+            # 全ての Exception を再表示
+            for i, opt_exception in enumerate(opt_exceptions):
+                if opt_exception is not None:
+                    print()
+                    print(f'===== unexpected exception raised on worker {i} =====')
+                    print_exception(opt_exception)
+                    print()
+
+            # monitor worker を残してユーザーが結果を確認できるようにする
+            if confirm_before_exit:
+                print()
+                print('='*len(Msg.CONFIRM_BEFORE_EXIT))
+                print(Msg.CONFIRM_BEFORE_EXIT)
+                print('='*len(Msg.CONFIRM_BEFORE_EXIT))
+                input()
+
+            return self.history.get_df()  # with 文を抜けると actor は消えるが .copy() はこの段階では不要
+
+    @staticmethod
+    def terminate_all():
+        warnings.warn(
+            "terminate_all() is deprecated and will be removed in a future version. "
+            "In current and later versions, the equivalent of terminate_all() will be executed when optimize() finishes. "
+            "Therefore, you can simply remove terminate_all() from your code. "
+            "If you want to stop program before terminating monitor process, "
+            "use ``confirm_before_exit`` argument like ``FEMOpt.optimize(confirm_before_exit=True)``",
+            DeprecationWarning,
+            stacklevel=2
+        )
 
 
 def _start_monitor_server(