pyfemtet 0.7.0__py3-none-any.whl → 0.8.0__py3-none-any.whl

pyfemtet/_util/excel_parse_util.py

@@ -0,0 +1,138 @@
+import os
+import numpy as np
+import pandas as pd
+
+
+__all__ = [
+    'ParseAsParameter',
+    'ParseAsConstraint',
+    'ParseAsObjective',
+    'search_index',
+    'search_r',
+    'search_c',
+]
+
+
+def parse_excel(book_path, sheet_name, keyword, required, optional) -> pd.DataFrame:
+    """Excel シートからパラメータを取得します。
+
+    シートのパースプロセスは以下の通りです。
+
+    1. シート全体 (A1 セルから、値が入力されている最終セルまで) をデータに取り込みます。
+    2. すべてのセルが空白である列をデータから除きます。
+    3. すべてのセルが空白である行をデータから除きます。
+    4. 最も左上（上が優先）にある keyword に一致するセルより上および左の行・列をデータから除きます。
+
+    Args:
+        book_path: Excel book のパス。
+        sheet_name: シート名。
+        keyword (str): 必ず含まれるべき、表データの最初の列名として使う文字列。
+        required (list[str]): 必ず含まれるべき、表データの列名として使う文字列のリスト。
+        optional (list[str]): 表データの列名として使ってよい文字列のリスト。
+
+    Returns:
+
+    """
+
+    # 存在チェック
+    if not os.path.exists(book_path):
+        raise FileNotFoundError(book_path)
+
+    # 読み込み
+    df = pd.read_excel(book_path, sheet_name, header=None)
+
+    # NaN のみからなる列を削除する
+    valid_columns = [col for col in df.columns if df[col].notna().sum()]
+    df = df[valid_columns]
+
+    # NaN のみからなる行を削除する
+    valid_rows = [row for row in df.index if df.loc[row].notna().sum()]
+    df = df.loc[valid_rows]
+
+    # 「変数名」を左上とする表にする
+    df: pd.DataFrame
+    idx = np.where(df.values == keyword)
+    r = idx[0][0]
+    c = idx[1][0]
+    df = pd.DataFrame(df.iloc[1+r:, c:].values, columns=df.iloc[r, c:].values)
+
+    # パースが成功しているかチェックする
+    lack = True
+    for col in df.columns:
+        lack *= col == keyword
+        lack *= col in required
+    if lack:
+        raise RuntimeError(f'Some required keywords are lacked. '
+                           f'Required keywords are {keyword} (required and must be first) and '
+                           f'{required} (required), '
+                           f'and {optional} is optional.')
+
+    return df
+
+
+class ParseBase:
+    KEYWORD = ''
+    REQUIRED_COLUMNS = []
+    OPTIONAL_COLUMNS = []
+
+    @classmethod
+    def parse(cls, book_path, sheet_name) -> pd.DataFrame:
+        return parse_excel(book_path, sheet_name, cls.KEYWORD, cls.REQUIRED_COLUMNS, cls.OPTIONAL_COLUMNS)
+
+
+class ParseAsParameter(ParseBase):
+    name = '変数名'
+    value = '値'
+    lb = '下限'
+    ub = '上限'
+    step = 'ステップ'
+    use = '使用'
+    KEYWORD = name
+    REQUIRED_COLUMNS = [value, lb, ub]
+    OPTIONAL_COLUMNS = [step, use]
+
+
+class ParseAsObjective(ParseBase):
+    name = '目的名'
+    value = '値'
+    direction = '目標'
+    use = '使用'
+    KEYWORD = name
+    REQUIRED_COLUMNS = [value, direction]
+    OPTIONAL_COLUMNS = [use]
+
+
+class ParseAsConstraint(ParseBase):
+    name = '拘束名'
+    value = '値'
+    lb = '下限'
+    ub = '上限'
+    strict = '厳守'
+    use = '使用'
+    calc_before_solve = 'ソルブ前に計算'
+    KEYWORD = name
+    REQUIRED_COLUMNS = [value]
+    OPTIONAL_COLUMNS = [lb, ub, strict, use, calc_before_solve]
+
+
+def search_index(book_path, sheet_name, value):
+    df = pd.read_excel(book_path, sheet_name, header=None)
+    idx = np.where(df.values == value)
+    r = idx[0][0]
+    c = idx[1][0]
+    return r, c
+
+
+def search_r(book_path, sheet_name, value):
+    return search_index(book_path, sheet_name, value)[0]
+
+
+def search_c(book_path, sheet_name, value):
+    return search_index(book_path, sheet_name, value)[1]
+
+
+if __name__ == '__main__':
+    os.chdir(os.path.dirname(__file__))
+    print(ParseAsParameter.parse('sample.xlsx', 'Sheet1'))
+    print(search_r('sample.xlsx', 'Sheet1', 'X2'))
+    print(search_c('sample.xlsx', 'Sheet1', '値'))

pyfemtet/brep/__init__.py

@@ -1,3 +0,0 @@
-from pyfemtet.brep._impl import ModelUpdater
-
-__all__ = ['ModelUpdater']

pyfemtet/brep/_impl.py

@@ -1,7 +1,10 @@
+import warnings
+
+
 try:
     import brepmatching
+
 except ModuleNotFoundError as e:
-    import warnings
     warnings.warn(
         'There is no installation of `brepmatching`. '
         'Please confirm installation via '
@@ -10,5 +13,6 @@ except ModuleNotFoundError as e:
     )
     raise e
 
-from brepmatching.pyfemtet_scripts.replace_model_considering_their_matching import ModelUpdater
-
+except FileNotFoundError as e:
+    warnings.warn(str(e))
+    raise e

pyfemtet/opt/_femopt.py

@@ -12,7 +12,7 @@ from traceback import print_exception
 # 3rd-party
 import numpy as np
 import pandas as pd
-from dask.distributed import LocalCluster, Client
+from dask.distributed import LocalCluster, Client, get_worker, Nanny
 
 # pyfemtet relative
 from pyfemtet.opt.interface import FEMInterface, FemtetInterface
@@ -136,6 +136,7 @@ class FEMOpt:
         self.monitor_server_kwargs = dict()
         self.monitor_process_worker_name = None
         self._hv_reference = None
+        self._extra_space_dir = None
 
     # multiprocess 時に pickle できないオブジェクト参照の削除
     def __getstate__(self):
@@ -164,6 +165,7 @@ class FEMOpt:
             step: float = None,
             properties: dict[str, str or float] = None,
             pass_to_fem: bool = True,
+            fix: bool = False,
     ):
         # noinspection PyUnresolvedReferences
         """Adds a parameter to the optimization problem.
@@ -176,6 +178,15 @@ class FEMOpt:
             step (float, optional): The step of parameter. If specified, parameter is used as discrete. Defaults to None.
             properties (dict[str, str or float], optional): Additional information about the parameter. Defaults to None.
             pass_to_fem (bool, optional): If this variable is used directly in FEM model update or not. If False, this parameter can be just used as inpt of expressions. Defaults to True.
+            fix (bool, optiona):
+                パラメータを initial_value で固定します。
+                開発時にパラメータを振るか振らないかを
+                簡単に変更するための便利引数です。
+                True のとき、lower_bound, upper_bound, step, properties の
+                値は、有効かどうかのチェックには使われますが、最適化では
+                使われなくなります。
+                デフォルトは False です。
+
 
         Raises:
             ValueError: If initial_value is not specified and the value for the given name is also not specified in FEM.
@@ -208,16 +219,27 @@ class FEMOpt:
             if initial_value is None:
                 raise ValueError('initial_value を指定してください.')
 
-        prm = Parameter(
-            name=name,
-            value=float(initial_value),
-            lower_bound=float(lower_bound) if lower_bound is not None else None,
-            upper_bound=float(upper_bound) if upper_bound is not None else None,
-            step=float(step) if step is not None else None,
-            pass_to_fem=pass_to_fem,
-            properties=properties,
-        )
-        self.opt.variables.add_parameter(prm)
+        if not fix:
+            prm = Parameter(
+                name=name,
+                value=float(initial_value),
+                lower_bound=float(lower_bound) if lower_bound is not None else None,
+                upper_bound=float(upper_bound) if upper_bound is not None else None,
+                step=float(step) if step is not None else None,
+                pass_to_fem=pass_to_fem,
+                properties=properties,
+            )
+            self.opt.variables.add_parameter(prm)
+
+        else:
+            warnings.filterwarnings('ignore', category=UserWarning, message="The function 'add_expression' is experimental")
+            self.add_expression(
+                name=name,
+                fun=lambda: initial_value,
+                pass_to_fem=pass_to_fem,
+                properties=properties,
+            )
+
 
     @experimental_feature
     def add_expression(
@@ -298,7 +320,7 @@ class FEMOpt:
 
     def add_objective(
             self,
-            fun,
+            fun: callable or None = None,
             name: str or None = None,
             direction: str or float = 'minimize',
             args: tuple or None = None,
@@ -308,7 +330,7 @@ class FEMOpt:
         """Adds an objective to the optimization problem.
 
         Args:
-            fun (callable): The objective function.
+            fun (callable or None, optional): The objective function. This argument is optional but
             name (str or None, optional): The name of the objective. Defaults to None.
             direction (str or float, optional): The optimization direction. Varid values are 'maximize', 'minimize' or a float value. Defaults to 'minimize'.
             args (tuple or None, optional): Additional arguments for the objective function. Defaults to None.
@@ -343,6 +365,10 @@ class FEMOpt:
         """
 
         # 引数の処理
+        if fun is None:
+            from pyfemtet.opt.interface._surrogate._base import SurrogateModelInterfaceBase
+            if not isinstance(self.fem, SurrogateModelInterfaceBase):
+                raise ValueError('`fun` argument is not specified.')
         if args is None:
             args = tuple()
         elif not isinstance(args, tuple):
@@ -402,7 +428,6 @@ class FEMOpt:
                 kwargs=kwargs,
             )
 
-
     def add_constraint(
             self,
             fun,
@@ -611,6 +636,7 @@ class FEMOpt:
             if self.fem._load_problem_from_me:
                 self.fem.load_parameter(self.opt)
                 self.fem.load_objective(self.opt)
+                self.fem.load_constraint(self.opt)
 
         # resolve expression dependencies
         self.opt.variables.resolve()
@@ -675,10 +701,6 @@ class FEMOpt:
                     directions,
                 )
 
-            # Femtet の confirm_before_exit のセット
-            self.fem.confirm_before_exit = confirm_before_exit
-            self.fem.kwargs['confirm_before_exit'] = confirm_before_exit
-
             logger.info('Femtet loaded successfully.')
 
         # クラスターの設定
@@ -718,30 +740,43 @@ class FEMOpt:
             #   これは CLI の --no-nanny オプションも同様らしい。
 
             # クラスターの構築
+            # noinspection PyTypeChecker
             cluster = LocalCluster(
                 processes=True,
                 n_workers=n_parallel,
                 threads_per_worker=1,
+                worker_class=Nanny,
             )
             logger.info('LocalCluster launched successfully.')
 
-            self.client = Client(cluster, direct_to_workers=False)
-            self.scheduler_address = self.client.scheduler.address
+            self.client = Client(
+                cluster,
+                direct_to_workers=False,
+            )
             logger.info('Client launched successfully.')
 
-            # 最適化タスクを振り分ける worker を指定
-            subprocess_indices = list(range(n_parallel))[1:]
-            worker_addresses = list(self.client.nthreads().keys())
+            self.scheduler_address = self.client.scheduler.address
 
-            # monitor worker の設定
-            self.monitor_process_worker_name = worker_addresses[0]
-            worker_addresses[0] = 'Main'
+            # worker address を取得
+            nannies_dict: dict[Any, Nanny] = self.client.cluster.workers
+            nannies = tuple(nannies_dict.values())
+
+            # ひとつの Nanny を選んで monitor 用にしつつ
+            # その space は main process に使わせるために記憶する
+            self.monitor_process_worker_name = nannies[0].worker_address
+            self._extra_space_dir = nannies[0].worker_dir
+
+            # 名前と address がごちゃごちゃになっていて可読性が悪いが
+            # 選んだ以外の Nanny は計算を割り当てる用にする
+            worker_addresses = ['Main']
+            worker_addresses.extend([n.worker_address for n in nannies[1:]])
+            subprocess_indices = list(range(n_parallel))[1:]
 
         with self.client.cluster as _cluster, self.client as _client:
 
             # actor の設定
-            self.status = OptimizationStatus(_client)
-            self.worker_status_list = [OptimizationStatus(_client, name) for name in worker_addresses]  # tqdm 検討
+            self.status = OptimizationStatus(_client, worker_address=self.monitor_process_worker_name)
+            self.worker_status_list = [OptimizationStatus(_client, worker_address=self.monitor_process_worker_name, name=name) for name in worker_addresses]  # tqdm 検討
             self.status.set(OptimizationStatus.SETTING_UP)
             self.history = History(
                 self.history_path,
@@ -773,7 +808,6 @@ class FEMOpt:
             logger.info('Process monitor initialized successfully.')
 
             # fem
-            # TODO: n_parallel=1 のときもアップロードしている。これを使うべきか、アップロードしないべき。
             self.fem._setup_before_parallel(_client)
 
             # opt
@@ -794,7 +828,7 @@ class FEMOpt:
                 subprocess_indices,
                 [self.worker_status_list] * len(subprocess_indices),
                 [wait_setup] * len(subprocess_indices),
-                workers=worker_addresses,
+                workers=worker_addresses if self.opt.is_cluster else worker_addresses[1:],
                 allow_other_workers=False,
             )
 
@@ -818,6 +852,7 @@ class FEMOpt:
                     ),
                     kwargs=dict(
                         skip_reconstruct=True,
+                        space_dir=self._extra_space_dir,
                     )
                 )
                 t_main.start()
@@ -882,7 +917,10 @@ class FEMOpt:
                 print('='*len(Msg.CONFIRM_BEFORE_EXIT))
                 input()
 
-            return self.history.get_df()  # with 文を抜けると actor は消えるが .copy() はこの段階では不要
+            df = self.history.get_df()  # with 文を抜けると actor は消えるが .copy() はこの段階では不要
+
+        return df
+
 
     @staticmethod
     def terminate_all():

pyfemtet/opt/_femopt_core.py

@@ -262,6 +262,8 @@ def is_feasible(value, lb, ub):
     Returns:
         bool: True if the value satisfies the bounds; False otherwise.
     """
+    if np.isnan(value):
+        return False
     if lb is None and ub is not None:
         return value <= ub
     elif lb is not None and ub is None:
@@ -290,10 +292,11 @@ class Function:
         # COM 定数を一度 _Scapegoat 型のオブジェクトにする
         # ParametricIF で使う dll 関数は _FuncPtr 型であって __globals__ を持たないが、
         # これは絶対に constants を持たないので単に無視すればよい。
-        if not isinstance(fun, ctypes._CFuncPtr):
-            for varname in fun.__globals__:
-                if isinstance(fun.__globals__[varname], Constants):
-                    fun.__globals__[varname] = _Scapegoat()
+        if fun is not None:
+            if not isinstance(fun, ctypes._CFuncPtr):
+                for varname in fun.__globals__:
+                    if isinstance(fun.__globals__[varname], Constants):
+                        fun.__globals__[varname] = _Scapegoat()
 
         self.fun = fun
         self.name = name
@@ -309,6 +312,9 @@ class Function:
         Returns:
             float
         """
+        if self.fun is None:
+            RuntimeError(f'`fun` of {self.name} is not specified.')
+
         args = self.args
         # Femtet 特有の処理
         if isinstance(fem, FemtetInterface):
@@ -318,11 +324,12 @@ class Function:
     def _restore_constants(self):
         """Helper function for parallelize Femtet."""
         fun = self.fun
-        if not isinstance(fun, ctypes._CFuncPtr):
-            for varname in fun.__globals__:
-                if isinstance(fun.__globals__[varname], _Scapegoat):
-                    if not fun.__globals__[varname]._ignore_when_restore_constants:
-                        fun.__globals__[varname] = constants
+        if fun is not None:
+            if not isinstance(fun, ctypes._CFuncPtr):
+                for varname in fun.__globals__:
+                    if isinstance(fun.__globals__[varname], _Scapegoat):
+                        if not fun.__globals__[varname]._ignore_when_restore_constants:
+                            fun.__globals__[varname] = constants
 
 
 class Objective(Function):
@@ -621,16 +628,30 @@ class History:
 
         self.set_df(df)
 
-    def get_df(self) -> pd.DataFrame:
+    def filter_valid(self, df_, keep_trial_num=False):
+        buff = df_[self.obj_names].notna()
+        idx = buff.prod(axis=1).astype(bool)
+        filtered_df = df_[idx]
+        if not keep_trial_num:
+            filtered_df.loc[:, 'trial'] = np.arange(len(filtered_df)) + 1
+        return filtered_df
+
+    def get_df(self, valid_only=False) -> pd.DataFrame:
         if self.__scheduler_address is None:
-            return self._df
+            if valid_only:
+                return self.filter_valid(self._df)
+            else:
+                return self._df
         else:
             # scheduler がまだ存命か確認する
             try:
                 with Lock('access-df'):
                     client_: 'Client' = get_client(self.__scheduler_address)
                     if 'df' in client_.list_datasets():
-                        return client_.get_dataset('df')
+                        if valid_only:
+                            return self.filter_valid(client_.get_dataset('df'))
+                        else:
+                            return client_.get_dataset('df')
                     else:
                         logger.debug('Access df of History before it is initialized.')
                         return pd.DataFrame()
@@ -806,17 +827,19 @@ class History:
         df['non_domi'] = False
 
         # feasible のものに non_domi の評価結果を代入する
-        df.loc[idx, 'non_domi'] = non_domi
+        if len(non_domi) > 0:
+            df.loc[idx, 'non_domi'] = non_domi
 
     def _calc_hypervolume(self, objectives, df):
 
+        # 単目的最適化ならば 0 埋めして終了
         if len(objectives) < 2:
             df.loc[len(df) - 1, 'hypervolume'] = 0.
             return
 
         # 最小化問題に変換された objective values を取得
         raw_objective_values = df[self.obj_names].values
-        objective_values = np.empty_like(raw_objective_values)
+        objective_values = np.full_like(raw_objective_values, np.nan)
         for n_trial in range(len(raw_objective_values)):
             for obj_idx, (_, objective) in enumerate(objectives.items()):
                 objective_values[n_trial, obj_idx] = objective.convert(raw_objective_values[n_trial, obj_idx])
@@ -830,13 +853,20 @@ class History:
             pareto_set_ = np.empty((0, len(self.obj_names)))
             for i in range(len(objective_values_)):
                 target = objective_values_[i]
-                dominated = False
-                # TODO: Array の計算に直して高速化する
-                for j in range(len(objective_values_)):
-                    compare = objective_values_[j]
-                    if all(target > compare):
-                        dominated = True
-                        break
+
+                if any(np.isnan(target)):
+                    # infeasible な場合 pareto_set の計算に含めない
+                    dominated = True
+
+                else:
+                    dominated = False
+                    # TODO: Array の計算に直して高速化する
+                    for j in range(len(objective_values_)):
+                        compare = objective_values_[j]
+                        if all(target > compare):
+                            dominated = True
+                            break
+
                 if not dominated:
                     pareto_set_ = np.concatenate([pareto_set_, [target]], axis=0)
 
@@ -848,34 +878,63 @@ class History:
             else:
                 return pareto_set_
 
+        def get_valid_worst_converted_objective_values(objective_values_: np.ndarray) -> np.ndarray:
+            # objective_values.max(axis=0)
+            ret = []
+            for row in objective_values_:
+                if not any(np.isnan(row)):
+                    ret.append(row)
+            return np.array(ret).max(axis=0)
+
         if self._hv_reference == 'dynamic-pareto':
             pareto_set, pareto_set_list = get_pareto(objective_values, with_partial=True)
             for i, partial_pareto_set in enumerate(pareto_set_list):
-                ref_point = pareto_set.max(axis=0) + 1e-8
-                hv = compute_hypervolume(partial_pareto_set, ref_point)
-                df.loc[i, 'hypervolume'] = hv
+                # 並列計算時など Valid な解がまだ一つもない場合は pareto_set が長さ 0 になる
+                # その場合 max() を取るとエラーになる
+                if len(pareto_set) == 0:
+                    df.loc[i, 'hypervolume'] = 0
+                else:
+                    ref_point = pareto_set.max(axis=0) + 1e-8
+                    hv = compute_hypervolume(partial_pareto_set, ref_point)
+                    df.loc[i, 'hypervolume'] = hv
             return
 
         elif self._hv_reference == 'dynamic-nadir':
             _, pareto_set_list = get_pareto(objective_values, with_partial=True)
             for i, partial_pareto_set in enumerate(pareto_set_list):
-                ref_point = objective_values.max(axis=0) + 1e-8
-                hv = compute_hypervolume(partial_pareto_set, ref_point)
-                df.loc[i, 'hypervolume'] = hv
+                # filter valid objective values only
+                values = get_valid_worst_converted_objective_values(objective_values)
+
+                # 並列計算時など Valid な解がまだ一つもない場合は長さ 0 になる
+                # その場合 max() を取るとエラーになる
+                if len(values) == 0:
+                    df.loc[i, 'hypervolume'] = 0
+
+                else:
+                    ref_point = values.max(axis=0) + 1e-8
+                    hv = compute_hypervolume(partial_pareto_set, ref_point)
+                    df.loc[i, 'hypervolume'] = hv
             return
 
         elif self._hv_reference == 'nadir':
             pareto_set = get_pareto(objective_values)
-            ref_point = objective_values.max(axis=0) + 1e-8
-            hv = compute_hypervolume(pareto_set, ref_point)
-            df.loc[len(df) - 1, 'hypervolume'] = hv
+            values = get_valid_worst_converted_objective_values(objective_values)
+            if len(values) == 0:
+                df.loc[len(df) - 1, 'hypervolume'] = 0
+            else:
+                ref_point = values.max(axis=0) + 1e-8
+                hv = compute_hypervolume(pareto_set, ref_point)
+                df.loc[len(df) - 1, 'hypervolume'] = hv
             return
 
         elif self._hv_reference == 'pareto':
             pareto_set = get_pareto(objective_values)
-            ref_point = pareto_set.max(axis=0) + 1e-8
-            hv = compute_hypervolume(pareto_set, ref_point)
-            df.loc[len(df) - 1, 'hypervolume'] = hv
+            if len(pareto_set) == 0:
+                df.loc[len(df) - 1, 'hypervolume'] = 0
+            else:
+                ref_point = pareto_set.max(axis=0) + 1e-8
+                hv = compute_hypervolume(pareto_set, ref_point)
+                df.loc[len(df) - 1, 'hypervolume'] = hv
             return
 
         elif (
@@ -935,7 +994,7 @@ class History:
         study = optuna.create_study(**kwargs)
 
         # add trial to study
-        df: pd.DataFrame = self.get_df()
+        df: pd.DataFrame = self.get_df(valid_only=True)
         for i, row in df.iterrows():
             FD = optuna.distributions.FloatDistribution
             kwargs = dict(
@@ -980,8 +1039,13 @@ class OptimizationStatus:
     TERMINATE_ALL = 60
     CRASHED = 70
 
-    def __init__(self, client, name='entire'):
-        self._future = client.submit(_OptimizationStatusActor, actor=True)
+    def __init__(self, client, worker_address, name='entire'):
+        self._future = client.submit(
+            _OptimizationStatusActor,
+            actor=True,
+            workers=[worker_address],
+            allow_other_workers=False,
+        )
         self._actor = self._future.result()
         self.name = name
         self.set(self.INITIALIZING)