pyfemtet 0.9.5__py3-none-any.whl → 1.0.0__py3-none-any.whl

pyfemtet/opt/optimizer/scipy_optimizer/__init__.py

@@ -0,0 +1 @@
+from ._scipy_optimizer import ScipyOptimizer

pyfemtet/opt/optimizer/scipy_optimizer/_scipy_optimizer.py

@@ -0,0 +1,383 @@
+from __future__ import annotations
+
+from typing import Callable
+
+from contextlib import suppress
+
+import numpy as np
+from scipy.optimize import minimize, OptimizeResult
+from scipy.optimize import NonlinearConstraint
+
+from pyfemtet._i18n import Msg, _
+from pyfemtet._util.closing import closing
+from pyfemtet.opt.problem.variable_manager import *
+from pyfemtet.opt.problem.problem import *
+from pyfemtet.opt.exceptions import *
+from pyfemtet.logger import get_module_logger
+
+from pyfemtet.opt.optimizer._base_optimizer import *
+
+
+__all__ = [
+    'ScipyOptimizer',
+]
+
+
+logger = get_module_logger('opt.optimizer', False)
+
+
+class _ScipyCallback:
+
+    def __init__(self, opt: ScipyOptimizer):
+        self.opt = opt
+
+    def __call__(self, xk: np.ndarray = None, intermediate_result: OptimizeResult = None):
+        pass
+
+
+class ScipyOptimizer(AbstractOptimizer):
+    """
+    Optimizer class that utilizes SciPy optimization methods.
+
+    This class serves as a wrapper around SciPy's optimization routines,
+    allowing customization of the optimization method, tolerance, and options.
+    It also provides mechanisms for handling constraints with enhancement and scaling.
+
+    Attributes:
+        method (str): The optimization method to use (e.g., 'BFGS', 'Nelder-Mead').
+        tol (float or None): Tolerance for termination.
+        options (dict): Additional options to pass to the SciPy optimizer.
+        constraint_enhancement (float): Small value added to enhance constraint handling.
+        constraint_scaling (float): Scaling factor applied to constraints.
+
+    Args:
+        method (str): The optimization method to use (e.g., 'BFGS', 'Nelder-Mead').
+        tol (float or None): Tolerance for termination.
+        
+    """
+
+    _timeout: None = None
+    _n_trials: None = None
+
+    def __init__(self, method: str = None, tol=None):
+        super().__init__()
+
+        self.method = method
+        self.tol = tol
+        self.options = {}
+        self.constraint_enhancement = 0.001
+        self.constraint_scaling = 1.
+
+    @property
+    def timeout(self):
+        return self._timeout
+
+    @timeout.setter
+    def timeout(self, value):
+        if value is not None:
+            raise NotImplementedError(_(
+                en_message='`ScipyOptimizer` cannot use timeout.',
+                jp_message='`ScipyOptimizer` では timeout は指定できません。'
+            ))
+
+    @property
+    def n_trials(self):
+        return self._n_trials
+
+    @n_trials.setter
+    def n_trials(self, value):
+        if value is not None:
+            raise NotImplementedError(_(
+                en_message='`ScipyOptimizer` cannot use n_trials.',
+                jp_message='`ScipyOptimizer` では n_trials は指定できません。'
+            ))
+
+    def _get_x0(self) -> np.ndarray:
+
+        # params を取得
+        params: dict[str, Parameter] = self.variable_manager.get_variables(
+            filter='parameter', format='raw'
+        )
+
+        for param in params.values():
+            if isinstance(param, CategoricalVariable):
+                raise NotImplementedError(_(
+                    en_message='Scipy can optimize only numerical parameters.',
+                    jp_message='Scipy では数値パラメータのみ最適化できます。'
+                ))
+
+        # params のうち fix == True のものを除く
+        x0 = np.array([p.value for p in params.values() if not p.properties.get('fix', False)])
+
+        return x0
+
+    def _warn_bounds_for_nelder_mead(self) -> None:
+        # https://github.com/scipy/scipy/issues/19991
+
+        if self.method.lower() != 'nelder-mead':
+            return
+
+        bounds = self._get_scipy_bounds()
+        if bounds is None:
+            return
+
+        x0 = self._get_x0()
+        if (np.allclose(x0, bounds[:, 0])
+                or np.allclose(x0, bounds[:, 1])):
+            logger.warning(Msg.WARN_SCIPY_NELDER_MEAD_BOUND)
+
+    def _setup_before_parallel(self):
+
+        if not self._done_setup_before_parallel:
+
+            super()._setup_before_parallel()  # flag inside
+
+            self._warn_bounds_for_nelder_mead()
+
+    def _get_scipy_bounds(self) -> np.ndarray | None:
+
+        has_any_bound = False
+
+        params: dict[str, Parameter] = self.variable_manager.get_variables(filter='parameter')
+        bounds = []
+        for param in params.values():
+            assert isinstance(param, NumericParameter)
+            bounds.append([
+                param.lower_bound or -np.inf,
+                param.upper_bound or np.inf,
+            ])
+            has_any_bound += (
+                    (param.lower_bound is not None)
+                    or (param.upper_bound is not None))
+
+        if has_any_bound:
+            bounds = np.array(bounds)
+        else:
+            bounds = None
+
+        return bounds
+
+    def _update_vm_by_xk(self, xk):
+
+        vm = self.variable_manager
+
+        # check interruption
+        self._check_and_raise_interruption()
+
+        # parameter suggestion
+        params = vm.get_variables(filter='parameter')
+        xk_list = list(xk)
+        for name, prm in params.items():
+
+            if prm.properties.get('fix', False):  # default is False
+                continue
+
+            if isinstance(prm, NumericParameter):
+                prm.value = xk_list.pop(0)
+
+            elif isinstance(prm, CategoricalParameter):
+                raise NotImplementedError(Msg.ERR_SCIPY_NOT_IMPLEMENT_CATEGORICAL)
+
+            else:
+                raise NotImplementedError
+        assert len(xk_list) == 0
+
+        # evaluate expressions
+        vm.eval_expressions()
+
+        # check interruption
+        self._check_and_raise_interruption()
+
+    def _scipy_constraint_fun(self, xk, cns: Constraint):
+
+        self._update_vm_by_xk(xk)
+
+        # update fem (very slow!)
+        if cns.using_fem:
+            logger.warning(Msg.WARN_USING_FEM_IN_NLC)
+            pass_to_fem = self.variable_manager.get_variables(filter='pass_to_fem')
+            self.fem.update_parameter(pass_to_fem)
+
+        return cns.eval(self.fem)
+
+    def _get_scipy_constraints(self) -> (
+        None
+        | list[NonlinearConstraint | dict]
+    ):
+        if len(self.constraints) == 0:
+            return None
+
+        if self.method is None:
+            method = 'SLSQP'
+        else:
+            method = self.method
+        assert method.lower() in ('cobyla', 'cobyqa', 'slsqp', 'trust-constr')
+
+        out = []
+        for cns in self.constraints.values():
+
+            # use Constraint object
+            if method.lower() in ('trust-constr', 'cobyqa'):
+
+                if cns.hard:
+                    raise NotImplementedError(
+                        Msg.F_ERR_SCIPY_METHOD_NOT_IMPLEMENT_HARD_CONSTRAINT(
+                            method
+                        )
+                    )
+
+                # constraint_scaling を使うためには violation を計算しなければならない
+                # TODO: 上下両端が決められている場合は二回計算することになるのでそれを解消する
+                if cns.lower_bound is not None:
+                    scipy_cns = NonlinearConstraint(
+                        fun=(
+                            lambda xk_, cns_=cns:
+                                (
+                                    cns.lower_bound
+                                    - self._scipy_constraint_fun(xk_, cns_)
+                                ) * self.constraint_scaling
+                                + self.constraint_enhancement
+                        ),
+                        lb=-np.inf,
+                        ub=0,
+                        keep_feasible=cns.hard,
+                        finite_diff_rel_step=self.options.get('finite_diff_rel_step', None),
+                    )
+                    out.append(scipy_cns)
+                if cns.upper_bound is not None:
+                    scipy_cns = NonlinearConstraint(
+                        fun=(
+                            lambda xk_, cns_=cns:
+                                (
+                                    self._scipy_constraint_fun(xk_, cns_)
+                                    - cns.upper_bound
+                                ) * self.constraint_scaling
+                                + self.constraint_enhancement
+                        ),
+                        lb=-np.inf,
+                        ub=0,
+                        keep_feasible=cns.hard,
+                        finite_diff_rel_step=self.options.get('finite_diff_rel_step', None),
+                    )
+                    out.append(scipy_cns)
+
+            # scipy_cns = NonlinearConstraint(
+                #     fun=lambda xk_, cns_=cns: self._scipy_constraint_fun(xk_, cns_),
+                #     lb=(cns.lower_bound or -np.inf) + self.constraint_enhancement,
+                #     ub=(cns.upper_bound or np.inf) - self.constraint_enhancement,
+                #     keep_feasible=cns.hard,
+                #     finite_diff_rel_step=self.options.get('finite_diff_rel_step', None),
+                # )
+                # out.append(scipy_cns)
+
+            # use dict object
+            else:
+
+                if method.lower() == 'slsqp' and not cns.hard:
+                    logger.warning(Msg.WARN_SCIPY_SLSQP_CANNOT_PROCESS_SOFT_CONSTRAINT)
+
+                if method.lower() == 'cobyla' and cns.hard:
+                    logger.error(
+                        Msg.F_ERR_SCIPY_METHOD_NOT_IMPLEMENT_HARD_CONSTRAINT(
+                            method))
+                    raise NotImplementedError(
+                        Msg.F_ERR_SCIPY_METHOD_NOT_IMPLEMENT_HARD_CONSTRAINT(
+                            method))
+
+                if cns.lower_bound is not None:
+
+                    scipy_cns = dict(
+                        type='ineq',
+                        fun=(lambda xk_, cns_=cns:
+                             (
+                                 self._scipy_constraint_fun(xk_, cns_)
+                                 - cns_.lower_bound
+                             ) * self.constraint_scaling
+                             - self.constraint_enhancement),
+                    )
+                    out.append(scipy_cns)
+
+                if cns.upper_bound is not None:
+                    scipy_cns = dict(
+                        type='ineq',
+                        fun=(lambda xk_, cns_=cns:
+                             (
+                                 cns_.upper_bound
+                                 - self._scipy_constraint_fun(xk_, cns_)
+                             ) * self.constraint_scaling
+                             - self.constraint_enhancement),
+                    )
+                    out.append(scipy_cns)
+
+        return out
+
+    def _get_scipy_callback(self) -> (
+        Callable[[OptimizeResult, ...], ...]
+        | Callable[[np.ndarray, ...], ...]
+    ):
+        return _ScipyCallback(self)
+
+    class _SolveSet(AbstractOptimizer._SolveSet):
+
+        def _hard_constraint_handling(self, e: HardConstraintViolation):
+            raise NotImplementedError(
+                Msg.ERR_SCIPY_HARD_CONSTRAINT_VIOLATION
+            ) from e
+
+        def _hidden_constraint_handling(self, e: _HiddenConstraintViolation):
+            raise NotImplementedError(
+                Msg.ERR_SCIPY_HIDDEN_CONSTRAINT
+            ) from e
+
+        def _skip_handling(self, e: SkipSolve):
+            raise NotImplementedError(
+                Msg.ERR_SCIPY_NOT_IMPLEMENT_SKIP
+            ) from e
+
+    def _objective(self, xk: np.ndarray) -> float:
+
+        with self._logging():
+
+            vm = self.variable_manager
+
+            # parameter suggestion
+            self._update_vm_by_xk(xk)
+
+            # construct TrialInput
+            x = vm.get_variables(filter='parameter')
+            x_pass_to_fem: dict[str, SupportedVariableTypes] = vm.get_variables(
+                filter='pass_to_fem', format='dict')
+
+            # process main fidelity model
+            solve_set = self._get_solve_set()
+            f_return = solve_set.solve(x, x_pass_to_fem)
+            assert f_return is not None
+            dict_y_internal = f_return[1]
+            y_internal: float = tuple(dict_y_internal.values())[0]  # type: ignore
+
+            return y_internal
+
+    def run(self):
+
+        # ===== finalize =====
+        self._finalize()
+
+        # ===== construct x0 =====
+        x0 = self._get_x0()
+
+        # ===== run =====
+        with closing(self.fem):
+
+            with self._setting_status(), suppress(InterruptOptimization):
+
+                minimize(
+                    self._objective,
+                    x0,
+                    args=(),
+                    method=self.method,
+                    bounds=self._get_scipy_bounds(),
+                    constraints=self._get_scipy_constraints(),
+                    tol=self.tol,
+                    callback=self._get_scipy_callback(),
+                    options=self.options,
+                )

pyfemtet/opt/prediction/__init__.py

@@ -0,0 +1,7 @@
+from ._model import *
+
+__all__ = [
+    'PyFemtetModel',
+    'AbstractModel',
+    'SingleTaskGPModel',
+]

pyfemtet/opt/prediction/_botorch_utils.py

@@ -0,0 +1,133 @@
+# import
+from __future__ import annotations
+
+from packaging import version
+
+import torch
+
+from gpytorch.mlls import ExactMarginalLogLikelihood
+from gpytorch.kernels import MaternKernel, ScaleKernel  # , RBFKernel
+from gpytorch.priors.torch_priors import GammaPrior  # , LogNormalPrior
+# from gpytorch.constraints.constraints import GreaterThan
+
+from botorch.models import SingleTaskGP
+from botorch.models.transforms import Standardize, Normalize
+
+# import fit_gpytorch_mll
+import botorch.version
+if version.parse(botorch.version.version) < version.parse("0.8.0"):
+    # noinspection PyUnresolvedReferences
+    from botorch.fit import fit_gpytorch_model as fit_gpytorch_mll
+
+else:
+    from botorch.fit import fit_gpytorch_mll
+
+
+__all__ = [
+    'get_standardizer_and_no_noise_train_yvar',
+    'setup_yvar_and_standardizer',
+    'setup_gp',
+    'get_matern_kernel_with_gamma_prior_as_covar_module',
+]
+
+
+def get_standardizer_and_no_noise_train_yvar(Y: torch.Tensor):
+    import gpytorch
+
+    standardizer = Standardize(m=Y.shape[-1])
+    min_noise = gpytorch.settings.min_fixed_noise.value(Y.dtype)
+    standardizer.forward(Y)  # require to un-transform
+    _, YVar = standardizer.untransform(Y, min_noise * torch.ones_like(Y))
+
+    return YVar, standardizer
+
+
+def setup_yvar_and_standardizer(
+        Y_: torch.Tensor,
+        observation_noise_: str | float | None,
+) -> tuple[torch.Tensor | None, Standardize]:
+
+    standardizer_ = None
+    train_yvar_ = None
+    if isinstance(observation_noise_, str):
+        if observation_noise_.lower() == 'no':
+            train_yvar_, standardizer_ = get_standardizer_and_no_noise_train_yvar(Y_)
+        else:
+            raise NotImplementedError
+    elif isinstance(observation_noise_, float):
+        train_yvar_ = torch.full_like(Y_, observation_noise_)
+
+    standardizer_ = standardizer_ or Standardize(m=Y_.shape[-1])
+
+    return train_yvar_, standardizer_
+
+
+def _get_matern_kernel_with_gamma_prior(
+        ard_num_dims: int, batch_shape=None
+) -> ScaleKernel:
+    r"""Constructs the Scale-Matern kernel that is used by default by
+    several models. This uses a Gamma(3.0, 6.0) prior for the lengthscale
+    and a Gamma(2.0, 0.15) prior for the output scale.
+    """
+
+    # PoFBoTorch の要請: 観測のない点は std を大きくしたい
+
+    return ScaleKernel(
+        base_kernel=MaternKernel(
+            nu=2.5,
+            ard_num_dims=ard_num_dims,
+            batch_shape=batch_shape,
+            # lengthscale_prior=GammaPrior(3.0, 6.0),
+            lengthscale_prior=GammaPrior(1, 9.0),
+        ),
+        batch_shape=batch_shape,
+        # outputscale_prior=GammaPrior(2.0, 0.15),
+        outputscale_prior=GammaPrior(1.0, 0.15),
+    )
+
+
+def get_matern_kernel_with_gamma_prior_as_covar_module(
+        X: torch.Tensor,
+        Y: torch.Tensor,
+        nu: float = 2.5,
+        lengthscale_prior: GammaPrior = None,
+        outputscale_prior: GammaPrior = None,
+):
+
+    _input_batch_shape, _aug_batch_shape = SingleTaskGP.get_batch_dimensions(X, Y)
+    ard_num_dims = X.shape[-1]
+    batch_shape = _aug_batch_shape
+
+    return ScaleKernel(
+        base_kernel=MaternKernel(
+            nu=nu,
+            ard_num_dims=ard_num_dims,
+            batch_shape=batch_shape,
+            lengthscale_prior=lengthscale_prior or GammaPrior(3.0, 6.0),
+        ),
+        batch_shape=batch_shape,
+        outputscale_prior=outputscale_prior or GammaPrior(2.0, 0.15),
+    )
+
+
+def setup_gp(X, Y, bounds, observation_noise, lh_class=None, covar_module=None):
+
+    lh_class = lh_class or ExactMarginalLogLikelihood
+
+    train_yvar_, standardizer_ = setup_yvar_and_standardizer(
+        Y, observation_noise
+    )
+
+    model_ = SingleTaskGP(
+        X,
+        Y,
+        train_Yvar=train_yvar_,
+        input_transform=Normalize(d=X.shape[-1], bounds=bounds),
+        outcome_transform=standardizer_,
+        covar_module=covar_module,
+    )
+
+    mll_ = lh_class(model_.likelihood, model_)
+    fit_gpytorch_mll(mll_)
+
+    return model_

pyfemtet/opt/prediction/_gpytorch_modules_extension.py

@@ -0,0 +1,142 @@
+#!/usr/bin/env python3
+
+# This file is derived from the one of botorch.
+
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+
+r"""
+Pre-packaged kernels for bayesian optimization, including a Scale/Matern
+kernel that is well-suited to low-dimensional high-noise problems, and
+a dimension-agnostic RBF kernel without outputscale.
+
+References:
+
+.. [Hvarfner2024vanilla]
+    C. Hvarfner, E. O. Hellsten, L. Nardi,
+    Vanilla Bayesian Optimization Performs Great in High Dimensions.
+    In International Conference on Machine Learning, 2024.
+"""
+
+from math import log, sqrt
+from typing import Optional, Sequence, Union
+
+import torch
+from gpytorch.constraints.constraints import GreaterThan
+from gpytorch.kernels import MaternKernel, RBFKernel, ScaleKernel
+from gpytorch.likelihoods.gaussian_likelihood import GaussianLikelihood
+from gpytorch.priors.torch_priors import GammaPrior, LogNormalPrior
+
+MIN_INFERRED_NOISE_LEVEL = 1e-4
+SQRT2 = sqrt(2)
+SQRT3 = sqrt(3)
+
+
+# def get_matern_kernel_with_gamma_prior(
+#     ard_num_dims: int, batch_shape: Optional[torch.Size] = None
+# ) -> ScaleKernel:
+#     r"""Constructs the Scale-Matern kernel that is used by default by
+#     several models. This uses a Gamma(3.0, 6.0) prior for the lengthscale
+#     and a Gamma(2.0, 0.15) prior for the output scale.
+#     """
+#     return ScaleKernel(
+#         base_kernel=MaternKernel(
+#             nu=2.5,
+#             ard_num_dims=ard_num_dims,
+#             batch_shape=batch_shape,
+#             lengthscale_prior=GammaPrior(3.0, 6.0),
+#         ),
+#         batch_shape=batch_shape,
+#         outputscale_prior=GammaPrior(2.0, 0.15),
+#     )
+#
+#
+# def get_gaussian_likelihood_with_gamma_prior(
+#     batch_shape: Optional[torch.Size] = None,
+# ) -> GaussianLikelihood:
+#     r"""Constructs the GaussianLikelihood that is used by default by
+#     several models. This uses a Gamma(1.1, 0.05) prior and constrains the
+#     noise level to be greater than MIN_INFERRED_NOISE_LEVEL (=1e-4).
+#     """
+#     batch_shape = torch.Size() if batch_shape is None else batch_shape
+#     noise_prior = GammaPrior(1.1, 0.05)
+#     noise_prior_mode = (noise_prior.concentration - 1) / noise_prior.rate
+#     return GaussianLikelihood(
+#         noise_prior=noise_prior,
+#         batch_shape=batch_shape,
+#         noise_constraint=GreaterThan(
+#             MIN_INFERRED_NOISE_LEVEL,
+#             transform=None,
+#             initial_value=noise_prior_mode,
+#         ),
+#     )
+#
+#
+# def get_gaussian_likelihood_with_lognormal_prior(
+#     batch_shape: Optional[torch.Size] = None,
+# ) -> GaussianLikelihood:
+#     """Return Gaussian likelihood with a LogNormal(-4.0, 1.0) prior.
+#     This prior is based on [Hvarfner2024vanilla]_.
+#
+#     Args:
+#         batch_shape: Batch shape for the likelihood.
+#
+#     Returns:
+#         GaussianLikelihood with LogNormal(-4.0, 1.0) prior and constrains the
+#         noise level to be greater than MIN_INFERRED_NOISE_LEVEL (=1e-4).
+#     """
+#     batch_shape = torch.Size() if batch_shape is None else batch_shape
+#     noise_prior = LogNormalPrior(loc=-4.0, scale=1.0)
+#     return GaussianLikelihood(
+#         noise_prior=noise_prior,
+#         batch_shape=batch_shape,
+#         noise_constraint=GreaterThan(
+#             MIN_INFERRED_NOISE_LEVEL,
+#             transform=None,
+#             initial_value=noise_prior.mode,
+#         ),
+#     )
+
+
+def get_covar_module_with_dim_scaled_prior_extension(
+    ard_num_dims: int,
+    batch_shape: Optional[torch.Size] = None,
+    use_rbf_kernel: bool = True,
+    active_dims: Optional[Sequence[int]] = None,
+    loc_coef: float = 1.,
+    scale_coef: float = 1.,
+) -> Union[MaternKernel, RBFKernel]:
+    """Returns an RBF or Matern kernel with priors
+    from  [Hvarfner2024vanilla]_.
+
+    Args:
+        ard_num_dims: Number of feature dimensions for ARD.
+        batch_shape: Batch shape for the covariance module.
+        use_rbf_kernel: Whether to use an RBF kernel. If False, uses a Matern kernel.
+        active_dims: The set of input dimensions to compute the covariances on.
+            By default, the covariance is computed using the full input tensor.
+            Set this if you'd like to ignore certain dimensions.
+
+        loc_coef (float): The coefficient of loc of LogNormalPrior.
+        scale_coef (float): The coefficient of scale of LogNormalPrior.
+
+
+    Returns:
+        A Kernel constructed according to the given arguments. The prior is constrained
+        to have lengthscales larger than 0.025 for numerical stability.
+    """
+    base_class = RBFKernel if use_rbf_kernel else MaternKernel
+    lengthscale_prior = LogNormalPrior(loc=(SQRT2 + log(ard_num_dims) * 0.5) * loc_coef, scale=SQRT3 * scale_coef)
+    base_kernel = base_class(
+        ard_num_dims=ard_num_dims,
+        batch_shape=batch_shape,
+        lengthscale_prior=lengthscale_prior,
+        lengthscale_constraint=GreaterThan(
+            2.5e-2, transform=None, initial_value=lengthscale_prior.mode
+        ),
+        # pyre-ignore[6] GPyTorch type is unnecessarily restrictive.
+        active_dims=active_dims,
+    )
+    return base_kernel