mindspore 2.4.0__cp310-cp310-macosx_10_15_x86_64.whl

mindspore/scipy/optimize/_lagrange.py

@@ -0,0 +1,201 @@
+# Copyright 2022 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+"""Lagrange"""
+import sys
+import logging
+from typing import NamedTuple
+import numpy as np
+import mindspore as ms
+from mindspore import nn, ops, Parameter, Tensor, numpy as mnp
+from mindspore.nn import Cell
+from mindspore.scipy.utils import _to_scalar
+
+
+class _LagrangeResults(NamedTuple):
+    """Object holding optimization results.
+
+    Args:
+        succeeded (bool): return true if the solution satisfies the all constrains with respect to save_tol.
+        best_value (Tensor): best solution, has the same shape with x0.
+        object_value (float): the value of objective function at best_value.
+        loss_value (Tensor): the value of constrain functions at best_value.
+    """
+    succeeded: bool
+    best_value: float
+    object_value: float
+    loss_value: Tensor
+
+
+class AugmentLagrangeMethod(Cell):
+    """minimize lagrange"""
+
+    def __init__(self, objective_func, param, lower, upper, obj_weight, cons_num, constraints, coincide_fun, bound_val):
+        """Initialize Minimize lagrange."""
+        super(AugmentLagrangeMethod, self).__init__()
+        self.objective_func = objective_func
+        self.cons_num = cons_num
+        self.sigmoid = ops.Sigmoid()
+        self.relu = nn.ReLU()
+        self.pow = ops.Pow()
+        self.bound_val = bound_val
+        self.pow_rate = Tensor(2.0)
+        self.pow_rate_1 = Tensor(0.5)
+        if lower is not None:
+            self.lb = Tensor(lower)
+        if upper is not None:
+            self.ub = Tensor(upper)
+        self.param = Parameter(param)
+        self.obj_weight = Tensor(obj_weight)
+        self.sigma_k = Parameter(Tensor(1.0), requires_grad=False)
+        self.alpha = Tensor(0.1)
+        self.beta = Tensor(0.9)
+        self.epsilon_k = Parameter(1.0 / self.pow(self.sigma_k, self.alpha), requires_grad=False)
+        self.eta_k = Parameter(1.0 / self.sigma_k, requires_grad=False)
+        self.rho = Tensor(1.1)
+        self.constrain = constraints
+        self.mul_k = Parameter(mnp.ones((cons_num,), dtype=ms.float32), requires_grad=False)
+        self.loss_list = Parameter(mnp.ones((cons_num,), dtype=ms.float32), requires_grad=False)
+        self.object_value = Parameter(Tensor(1.0), requires_grad=False)
+        self.best_value = Parameter(param, requires_grad=False)
+        self.coincide_fun = coincide_fun
+
+
+    def normalize(self, x):
+        """normalize the input_x."""
+        if self.bound_val == 0:
+            return x
+        if self.bound_val == 1:
+            return (-x) * x + self.ub
+        if self.bound_val == 2:
+            return x * x + self.lb
+        return (self.ub - self.lb) * self.sigmoid(x) + self.lb
+
+
+    def get_loss(self, x):
+        """get the loss of function."""
+        x = self.normalize(x)
+        pernalty_func = 0
+        if self.coincide_fun is not None:
+            tmp_res = self.coincide_fun(x)
+            for i in range(self.cons_num):
+                cons = self.constrain[i]
+                self.loss_list[i] = cons(x, tmp_res)
+                pernalty_tmp = self.pow(self.relu(self.mul_k[i] / self.sigma_k + self.loss_list[i]), self.pow_rate)\
+                    - self.pow((self.mul_k[i] / self.sigma_k), self.pow_rate)
+                pernalty_func += pernalty_tmp
+            objective_val = self.objective_func(x, tmp_res)
+        else:
+            for i in range(self.cons_num):
+                cons = self.constrain[i]
+                self.loss_list[i] = cons(x)
+                pernalty_tmp = self.pow(self.relu(self.mul_k[i] / self.sigma_k + self.loss_list[i]), self.pow_rate)\
+                    - self.pow((self.mul_k[i] / self.sigma_k), self.pow_rate)
+                pernalty_func += pernalty_tmp
+            objective_val = self.objective_func(x)
+        loss1 = self.obj_weight * objective_val
+        lagrangian_func = loss1 + self.sigma_k / 2 * pernalty_func
+        res = [lagrangian_func, self.loss_list, objective_val, x]
+        return res
+
+
+    def update_param(self):
+        """update the parameter."""
+        v_k = 0
+        for i in range(self.cons_num):
+            v_k_i = self.pow(max(self.loss_list[i], -self.mul_k[i] / self.sigma_k), self.pow_rate)
+            v_k = v_k + v_k_i
+        v_k = self.pow(v_k, self.pow_rate_1)
+        if v_k <= self.epsilon_k:
+            for i in range(self.cons_num):
+                self.mul_k[i] = max(self.mul_k[i] + self.sigma_k * self.loss_list[i], 0)
+            self.eta_k = self.eta_k / self.sigma_k
+            self.epsilon_k = min(self.epsilon_k / (self.sigma_k ** self.beta), sys.float_info.max)
+        else:
+            self.sigma_k = self.rho * self.sigma_k
+            self.eta_k = 1.0 / self.sigma_k
+            self.epsilon_k = 1.0 / (self.sigma_k ** self.alpha)
+
+
+    def construct(self, x):
+        res = self.get_loss(self.param)
+        lang_func, self.loss_list, self.object_value, self.best_value = res
+        x = x * lang_func
+        return x
+
+
+class Loss(Cell):
+    """compute loss from data and label"""
+    def __init__(self):
+        """Initialize the loss."""
+        super(Loss, self).__init__()
+
+    def construct(self, data, label):
+        diff = data - label
+        return diff
+
+
+def check_tol(f0, object_value, save_tol, loss_list):
+    """check tolerance."""
+    chk_res = False
+    if (np.array(loss_list) - save_tol).max() < 0 and object_value < f0:
+        chk_res = True
+    return chk_res
+
+
+def minimize_lagrange(func, x0, constraints, save_tol=None, obj_weight=1.0, lower=None, upper=None,
+                      learning_rate=0.1, coincide_fun=None, rounds=10, steps=1000, log_sw=False):
+    """minimize lagrange."""
+    cons_num = len(constraints)
+    if lower is None and upper is None:
+        bound_val = 0
+    elif lower is None and upper is not None:
+        bound_val = 1
+    elif lower is not None and upper is None:
+        bound_val = 2
+    else:
+        bound_val = 3
+
+    net = AugmentLagrangeMethod(func, x0, lower, upper, obj_weight, cons_num, constraints, coincide_fun, bound_val)
+    loss = Loss()
+    optimizer = nn.Adam(params=net.trainable_params(), learning_rate=learning_rate, use_locking=True,
+                                                use_nesterov=False)
+    loss_net = nn.WithLossCell(net, loss)
+    label = Tensor(0.0, ms.float32)
+    image = Tensor(1.0, ms.float32)
+    train_net = nn.TrainOneStepCell(loss_net, optimizer)
+    f0 = 0
+    best_value = x0.asnumpy()
+    chk_res = False
+    loss_list = mnp.ones((cons_num,), dtype=ms.float32)
+    if save_tol is None:
+        save_tol = mnp.ones((cons_num,), dtype=ms.float32)
+    for j in range(rounds):
+        for i in range(steps):
+            lang_val = train_net(image, label)
+            if log_sw:
+                logging.info("round:%d, step:%d", j, i)
+                logging.info("object_value:%f, loss_value:%f, lagrange_value:%f", net.object_value.value(),
+                             net.loss_list.value(), lang_val)
+            if i == 0 and j == 0:
+                f0 = net.object_value.value().asnumpy()
+            chkres = check_tol(f0, net.object_value.value().asnumpy(), save_tol, net.loss_list.value().asnumpy())
+            if chkres:
+                chk_res = True
+                loss_list = net.loss_list.value().asnumpy()
+                best_value = net.best_value.value().asnumpy()
+                f0 = net.object_value.value().asnumpy()
+        net.update_param()
+    res = _LagrangeResults(succeeded=_to_scalar(chk_res), best_value=best_value, object_value=f0, loss_value=loss_list)
+    return res

mindspore/scipy/optimize/_lbfgs.py

@@ -0,0 +1,146 @@
+# Copyright 2022 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+"""LBFGS"""
+from mindspore import ops, Parameter, numpy as mnp
+from mindspore.nn import Cell
+from mindspore.scipy.utils import grad, _to_scalar
+from mindspore.scipy.optimize.gradient_optimization_algorithm import GradientOptimizationAlgorithm
+from mindspore.scipy.optimize.gradient_optimization_algorithm import GradientOptimizationResults
+
+
+class SearchVector(Cell):
+    """Calculate the search vector"""
+    def __init__(self, shape, dtype):
+        """Initialize SearchVector."""
+        super(SearchVector, self).__init__()
+        # create buffer to save history data to compute new search vector
+        zeros = ops.Zeros()
+        self._tensor_s = Parameter(zeros(shape, dtype))
+        self._tensor_y = Parameter(zeros(shape, dtype))
+        self._alpha_rate = Parameter(zeros(shape, dtype))
+        self._beta_rate = Parameter(zeros(shape, dtype))
+
+    def construct(self, tensor_q, s_k, y_k):
+        """Building a graph to computes the search vector"""
+        # If the change of curvature is too small, use the gradient as the search vector directly.
+        y_s = self._dot(y_k, s_k)
+        if y_s <= 1e-10:
+            return tensor_q
+
+        # save data as history, use to compute search vector
+        self._tensor_s = self._save_history_data(self._tensor_s, s_k)
+        self._tensor_y = self._save_history_data(self._tensor_y, y_k)
+        rho = y_s.inv()
+        self._alpha_rate = self._save_history_data(self._alpha_rate, rho * s_k)
+        self._beta_rate = self._save_history_data(self._beta_rate, rho * y_k)
+
+        alpha_tensor = ops.ZerosLike()(self._alpha_rate)
+        history_size = self._tensor_s.shape[0]
+        for index in range(history_size - 1, -1, -1):
+            alpha = self._dot(self._alpha_rate[index], tensor_q)
+            tensor_q = tensor_q - alpha * self._tensor_y[index]
+            alpha_tensor[index] = alpha
+
+        gamma = y_s / self._dot(y_k, y_k)
+        tensor_r = gamma * tensor_q
+        alpha_s = ops.mul(alpha_tensor, self._tensor_s)
+        for index in range(history_size):
+            beta = self._dot(self._beta_rate[index], tensor_r)
+            tensor_r = tensor_r + alpha_s[index] - beta * self._tensor_s[index]
+        return tensor_r
+
+    def _save_history_data(self, history_buffer, data):
+        """Remove the first history data, and move the other history data forward one position
+           put the new data into the last position of the history buffer.
+        """
+        for index in range(history_buffer.shape[0] - 1):
+            history_buffer[index] = history_buffer[index + 1]
+        history_buffer[-1] = data
+        return history_buffer
+
+    def _dot(self, vector_a, vector_b):
+        shape = vector_a.shape + (1,)
+        res = ops.MatMul(transpose_a=True)(vector_a.reshape(shape), vector_b.reshape(shape))
+        return res.reshape(res.shape[:-1])
+
+
+class AlgorithmLbfgs(GradientOptimizationAlgorithm):
+    """minimize AlgorithmLbfgs"""
+
+    def __init__(self, func, jac, history_size):
+        """Initialize AlgorithmLbfgs."""
+        super(AlgorithmLbfgs, self).__init__(func, jac)
+        self.history_size = history_size
+        self.search_vector = None
+
+    def gen_search_vector(self, s_k, y_k, state):
+        """Calculate the search vector for the next line search"""
+        state["search_vector"] = state.get("g_star")
+
+        if self.search_vector is None:
+            shape = (self.history_size,) + s_k.shape
+            self.search_vector = SearchVector(shape, s_k.dtype)
+
+        state["search_vector"] = self.search_vector(state.get("search_vector"), s_k, y_k)
+
+
+def minimize_lbfgs(func, x_0, jac=None, history_size=20, maxiter=None, norm=mnp.inf, gtol=1e-5, line_search_maxiter=10):
+    """Minimize a function using LBFGS.
+
+    Implements the LBFGS algorithm
+
+    Args:
+        fun (Callable): function of the form f(x) where x is a flat Tensor and returns a real
+            scalar. The function should be composed of operations with vjp defined.
+        x_0 (Tensor): initial guess.
+        jac (Callable, optional): method for computing the gradient vector.
+        if it is None, the gradient will be estimated with gradient of func.
+        if it is a callable, it should be a function that returns the gradient vector:
+          jac(x, *args) -> array_like, shape (n,)
+          where x is an array with shape (n,) and args is a tuple with the fixed parameters.
+        history_size (int, optional): size of buffer used to help to update inv hessian, Default: 20.
+        maxiter (int, optional): maximum number of iterations.
+        norm (float): order of norm for convergence check. Default inf.
+        gtol (float): terminates minimization when |grad|_norm < g_tol.
+        line_search_maxiter (int): maximum number of linesearch iterations.
+
+    Returns:
+        LBFGSResults, results from LBFGS optimization.
+
+    Supported Platforms:
+        ``GPU`` ``CPU``
+    """
+    if jac is None:
+        jac = grad(func)
+
+    if maxiter is None:
+        maxiter = ops.size(x_0) * 200
+
+    state = AlgorithmLbfgs(func, jac, history_size).construct(x_0, maxiter, norm, gtol, line_search_maxiter)
+    results = GradientOptimizationResults(converged=_to_scalar(state.get("converged")),
+                                          failed=_to_scalar(state.get("failed")),
+                                          k=_to_scalar(state.get("k")),
+                                          nfev=_to_scalar(state.get("nfev")),
+                                          ngev=_to_scalar(state.get("ngev")),
+                                          nhev=_to_scalar(state.get("nhev")),
+                                          x_k=state.get("x_k"),
+                                          f_k=_to_scalar(state.get("f_k")),
+                                          g_k=state.get("g_k"),
+                                          H_k=state.get("H_k"),
+                                          old_old_fval=_to_scalar(state.get("old_old_fval")),
+                                          status=_to_scalar(state.get("status")),
+                                          line_search_status=_to_scalar(state.get("line_search_status")))
+
+    return results

mindspore/scipy/optimize/gradient_optimization_algorithm.py

@@ -0,0 +1,168 @@
+# Copyright 2022 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+"""GradientOptimizationAlgorithm"""
+from abc import abstractmethod
+from typing import NamedTuple
+from mindspore import numpy as mnp
+from mindspore.common import Tensor
+from mindspore.scipy.optimize.line_search import LineSearch
+from mindspore.scipy.utils import _norm
+
+
+class GradientOptimizationResults(NamedTuple):
+    """Results from gradient optimization.
+
+    Arg
+        converged (bool): `True`` if minimization converged.
+        failed (bool): `True`` if line search failed.
+        k (int): the number of iterations of the BFGS update.
+        nfev (int): total number of objective evaluations performed.
+        ngev (int): total number of jacobian evaluations
+        nhev (int): total number of hessian evaluations
+        x_k (Tensor): containing the last argument value found during the search. If
+            the search converged, then this value is the argmin of the objective
+            function.
+        f_k (float): containing the value of the objective function at `x_k`. If the
+            search converged, then this is the (local) minimum of the objective
+            function.
+        g_k (Tensor): containing the gradient of the objective function at `x_k`. If
+            the search converged the l2-norm of this Tensor should be below the
+            tolerance.
+        H_k (Tensor): containing the inverse of the estimated Hessian.
+        old_old_fval (float): Function value for the point preceding x=x_k.
+        status (int): describing end state.
+        line_search_status (int): describing line search end state (only means
+            something if line search fails).
+    """
+    converged: bool
+    failed: bool
+    k: int
+    nfev: int
+    ngev: int
+    nhev: int
+    x_k: Tensor
+    f_k: float
+    g_k: Tensor
+    H_k: Tensor
+    old_old_fval: float
+    status: int
+    line_search_status: int
+
+
+class GradientOptimizationAlgorithm():
+    """Base class of gradient optimization algorithms"""
+
+    def __init__(self, func, jac):
+        """Initialize GradientOptimizationAlgorithm."""
+        self.func = func
+        self.jac = jac
+        self.line_search = LineSearch(func, jac)
+
+    @abstractmethod
+    def gen_search_vector(self, s_k, y_k, state):
+        """Calculate the search vector for the next line search"""
+
+    def construct(self, x_0, maxiter=None, norm=mnp.inf, gtol=1e-5, line_search_maxiter=10):
+        """Calculate the minimum value of the func by the gradient optimization algorithm"""
+        if maxiter is None:
+            maxiter = mnp.size(x_0) * 200
+
+        f_0 = self.func(x_0)
+        g_0 = self.jac(x_0)
+
+        state = {
+            "converged": _norm(g_0, ord_=mnp.inf) < gtol,
+            "failed": False,
+            "k": 0,
+            "nfev": 1,
+            "ngev": 1,
+            "nhev": 0,
+            "x_k": x_0,
+            "f_k": f_0,
+            "g_k": g_0,
+            "search_vector": g_0,
+            "H_k": None,
+            "old_old_fval": f_0 + _norm(g_0) / 2,
+            "a_star": None,
+            "phi_star": None,
+            "g_star": None,
+            "status": 0,
+            "line_search_status": 0
+        }
+
+        while state.get("k") < maxiter:
+            # search direction vector
+            sdv = -1 * state.get("search_vector")
+            line_search_results = self.line_search(state.get("x_k"),
+                                                   sdv,
+                                                   old_fval=state.get("f_k"),
+                                                   old_old_fval=state.get("old_old_fval"),
+                                                   gfk=state.get("g_k"),
+                                                   maxiter=line_search_maxiter)
+            if isinstance(line_search_results, tuple):
+                state["failed"] = mnp.logical_not(line_search_results[0]) or line_search_results[1]
+                state["nfev"] += line_search_results[6]
+                state["ngev"] += line_search_results[7]
+                state["a_star"] = line_search_results[8]
+                state["phi_star"] = line_search_results[9]
+                state["g_star"] = line_search_results[11]
+                state["line_search_status"] = line_search_results[12]
+            else:
+                state["failed"] = mnp.logical_not(line_search_results.get("done")) or line_search_results.get("failed")
+                state["nfev"] += line_search_results.get("nfev")
+                state["ngev"] += line_search_results.get("ngev")
+                state["a_star"] = line_search_results.get("a_star")
+                state["phi_star"] = line_search_results.get("phi_star")
+                state["g_star"] = line_search_results.get("g_star")
+                state["line_search_status"] = line_search_results.get("status")
+
+            if state.get("failed"):
+                break
+
+            s_k = state.get("a_star") * sdv
+            x_k = state.get("x_k") + s_k
+            f_k = state.get("phi_star")
+            g_k = state.get("g_star")
+            y_k = g_k - state.get("g_k")
+
+            state["old_old_fval"] = state.get("f_k")
+            state["converged"] = _norm(g_k, ord_=norm) < gtol
+            state["x_k"] = x_k
+            state["f_k"] = f_k
+            state["g_k"] = g_k
+
+            if state.get("converged"):
+                break
+
+            self.gen_search_vector(s_k, y_k, state)
+
+            # counter of iteration
+            state["k"] = state.get("k") + 1
+
+        status = mnp.where(
+            state.get("converged"),
+            0,  # converged
+            mnp.where(
+                state.get("k") == maxiter,
+                1,  # max iters reached
+                mnp.where(
+                    state.get("failed"),
+                    2 + state.get("line_search_status"),  # ls failed (+ reason)
+                    -1,  # undefined
+                )
+            )
+        )
+        state["status"] = status
+        return state