mindspore 2.0.0a0__cp37-none-any.whl → 2.0.0rc1__cp37-none-any.whl

mindspore/scipy/optimize/_lagrange.py

@@ -0,0 +1,200 @@
+# Copyright 2022 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+"""Lagrange"""
+import sys
+from typing import NamedTuple
+import numpy as np
+import mindspore as ms
+from mindspore import nn, ops, Parameter, Tensor, numpy as mnp
+from mindspore.nn import Cell
+from mindspore.scipy.utils import _to_scalar
+
+
+class _LagrangeResults(NamedTuple):
+    """Object holding optimization results.
+
+    Args:
+        succeeded (bool): return true if the solution satisfies the all constrains with respect to save_tol.
+        best_value (Tensor): best solution, has the same shape with x0.
+        object_value (float): the value of objective function at best_value.
+        loss_value (Tensor): the value of constrain functions at best_value.
+    """
+    succeeded: bool
+    best_value: float
+    object_value: float
+    loss_value: Tensor
+
+
+class AugmentLagrangeMethod(Cell):
+    """minimize lagrange"""
+
+    def __init__(self, objective_func, param, lower, upper, obj_weight, cons_num, constraints, coincide_fun, bound_val):
+        """Initialize Minimize lagrange."""
+        super(AugmentLagrangeMethod, self).__init__()
+        self.objective_func = objective_func
+        self.cons_num = cons_num
+        self.sigmoid = ops.Sigmoid()
+        self.relu = nn.ReLU()
+        self.pow = ops.Pow()
+        self.bound_val = bound_val
+        self.pow_rate = Tensor(2.0)
+        self.pow_rate_1 = Tensor(0.5)
+        if lower is not None:
+            self.lb = Tensor(lower)
+        if upper is not None:
+            self.ub = Tensor(upper)
+        self.param = Parameter(param)
+        self.obj_weight = Tensor(obj_weight)
+        self.sigma_k = Parameter(Tensor(1.0), requires_grad=False)
+        self.alpha = Tensor(0.1)
+        self.beta = Tensor(0.9)
+        self.epsilon_k = Parameter(1.0 / self.pow(self.sigma_k, self.alpha), requires_grad=False)
+        self.eta_k = Parameter(1.0 / self.sigma_k, requires_grad=False)
+        self.rho = Tensor(1.1)
+        self.constrain = constraints
+        self.mul_k = Parameter(mnp.ones((cons_num,), dtype=ms.float32), requires_grad=False)
+        self.loss_list = Parameter(mnp.ones((cons_num,), dtype=ms.float32), requires_grad=False)
+        self.object_value = Parameter(Tensor(1.0), requires_grad=False)
+        self.best_value = Parameter(param, requires_grad=False)
+        self.coincide_fun = coincide_fun
+
+
+    def normalize(self, x):
+        """normalize the input_x."""
+        if self.bound_val == 0:
+            return x
+        if self.bound_val == 1:
+            return (-x) * x + self.ub
+        if self.bound_val == 2:
+            return x * x + self.lb
+        return (self.ub - self.lb) * self.sigmoid(x) + self.lb
+
+
+    def get_loss(self, x):
+        """get the loss of function."""
+        x = self.normalize(x)
+        pernalty_func = 0
+        if self.coincide_fun is not None:
+            tmp_res = self.coincide_fun(x)
+            for i in range(self.cons_num):
+                cons = self.constrain[i]
+                self.loss_list[i] = cons(x, tmp_res)
+                pernalty_tmp = self.pow(self.relu(self.mul_k[i] / self.sigma_k + self.loss_list[i]), self.pow_rate)\
+                    - self.pow((self.mul_k[i] / self.sigma_k), self.pow_rate)
+                pernalty_func += pernalty_tmp
+            objective_val = self.objective_func(x, tmp_res)
+        else:
+            for i in range(self.cons_num):
+                cons = self.constrain[i]
+                self.loss_list[i] = cons(x)
+                pernalty_tmp = self.pow(self.relu(self.mul_k[i] / self.sigma_k + self.loss_list[i]), self.pow_rate)\
+                    - self.pow((self.mul_k[i] / self.sigma_k), self.pow_rate)
+                pernalty_func += pernalty_tmp
+            objective_val = self.objective_func(x)
+        loss1 = self.obj_weight * objective_val
+        lagrangian_func = loss1 + self.sigma_k / 2 * pernalty_func
+        res = [lagrangian_func, self.loss_list, objective_val, x]
+        return res
+
+
+    def update_param(self):
+        """update the parameter."""
+        v_k = 0
+        for i in range(self.cons_num):
+            v_k_i = self.pow(max(self.loss_list[i], -self.mul_k[i] / self.sigma_k), self.pow_rate)
+            v_k = v_k + v_k_i
+        v_k = self.pow(v_k, self.pow_rate_1)
+        if v_k <= self.epsilon_k:
+            for i in range(self.cons_num):
+                self.mul_k[i] = max(self.mul_k[i] + self.sigma_k * self.loss_list[i], 0)
+            self.eta_k = self.eta_k / self.sigma_k
+            self.epsilon_k = min(self.epsilon_k / (self.sigma_k ** self.beta), sys.float_info.max)
+        else:
+            self.sigma_k = self.rho * self.sigma_k
+            self.eta_k = 1.0 / self.sigma_k
+            self.epsilon_k = 1.0 / (self.sigma_k ** self.alpha)
+
+
+    def construct(self, x):
+        res = self.get_loss(self.param)
+        lang_func, self.loss_list, self.object_value, self.best_value = res
+        x = x * lang_func
+        return x
+
+
+class Loss(Cell):
+    """compute loss from data and label"""
+    def __init__(self):
+        """Initialize the loss."""
+        super(Loss, self).__init__()
+
+    def construct(self, data, label):
+        diff = data - label
+        return diff
+
+
+def check_tol(f0, object_value, save_tol, loss_list):
+    """check tolerance."""
+    chk_res = False
+    if (np.array(loss_list) - save_tol).max() < 0 and object_value < f0:
+        chk_res = True
+    return chk_res
+
+
+def minimize_lagrange(func, x0, constraints, save_tol=None, obj_weight=1.0, lower=None, upper=None,
+                      learning_rate=0.1, coincide_fun=None, rounds=10, steps=1000, log_sw=False):
+    """minimize lagrange."""
+    cons_num = len(constraints)
+    if lower is None and upper is None:
+        bound_val = 0
+    elif lower is None and upper is not None:
+        bound_val = 1
+    elif lower is not None and upper is None:
+        bound_val = 2
+    else:
+        bound_val = 3
+
+    net = AugmentLagrangeMethod(func, x0, lower, upper, obj_weight, cons_num, constraints, coincide_fun, bound_val)
+    loss = Loss()
+    optimizer = nn.Adam(params=net.trainable_params(), learning_rate=learning_rate, use_locking=True,
+                                                use_nesterov=False)
+    loss_net = nn.WithLossCell(net, loss)
+    label = Tensor(0.0, ms.float32)
+    image = Tensor(1.0, ms.float32)
+    train_net = nn.TrainOneStepCell(loss_net, optimizer)
+    f0 = 0
+    best_value = x0.asnumpy()
+    chk_res = False
+    loss_list = mnp.ones((cons_num,), dtype=ms.float32)
+    if save_tol is None:
+        save_tol = mnp.ones((cons_num,), dtype=ms.float32)
+    for j in range(rounds):
+        for i in range(steps):
+            lang_val = train_net(image, label)
+            if log_sw:
+                print(F"round:{j}, step:{i}")
+                print(F"object_value:{net.object_value.value()}, loss_value:{net.loss_list.value()}, \
+                            lagrange_value:{lang_val}")
+            if i == 0 and j == 0:
+                f0 = net.object_value.value().asnumpy()
+            chkres = check_tol(f0, net.object_value.value().asnumpy(), save_tol, net.loss_list.value().asnumpy())
+            if chkres:
+                chk_res = True
+                loss_list = net.loss_list.value().asnumpy()
+                best_value = net.best_value.value().asnumpy()
+                f0 = net.object_value.value().asnumpy()
+        net.update_param()
+    res = _LagrangeResults(succeeded=_to_scalar(chk_res), best_value=best_value, object_value=f0, loss_value=loss_list)
+    return res

mindspore/scipy/optimize/line_search.py

@@ -14,6 +14,7 @@
 # ============================================================================
 """line search"""
 from typing import NamedTuple
+from mindspore import ops
 from ... import nn
 from ... import numpy as mnp
 from ...common import dtype as mstype
@@ -151,7 +152,7 @@ def _zoom(fn, a_low, phi_low, dphi_low, a_high, phi_high, dphi_high, phi_0, g_0,
         state["phi_high"] = mnp.where(j_to_high, phi_j, state["phi_high"])
         state["dphi_high"] = mnp.where(j_to_high, dphi_j, state["dphi_high"])
 
-        j_to_star = mnp.logical_not(j_to_high) and mnp.abs(dphi_j) <= -c2 * dphi_0
+        j_to_star = mnp.logical_not(j_to_high) and mnp.abs(dphi_j) <= ops.negative(ops.add(c2, Tensor(0))) * dphi_0
         state["done"] = j_to_star
         state["a_star"] = mnp.where(j_to_star, a_j, state["a_star"])
         state["phi_star"] = mnp.where(j_to_star, phi_j, state["phi_star"])
@@ -326,7 +327,7 @@ def line_search(f, xk, pk, jac=None, gfk=None, old_fval=None, old_old_fval=None,
         LineSearchResults, results of line search results.
 
     Supported Platforms:
-        ``CPU`` ``GPU``
+        ``GPU`` ``CPU``
 
     Examples:
         >>> import numpy as onp

mindspore/scipy/optimize/minimize.py

@@ -18,6 +18,7 @@ from typing import NamedTuple
 from ...common import Tensor
 from ._bfgs import minimize_bfgs
 from ._lbfgs import minimize_lbfgs
+from ._lagrange import minimize_lagrange
 
 
 class OptimizeResults(NamedTuple):
@@ -47,6 +48,22 @@ class OptimizeResults(NamedTuple):
     nit: int
 
 
+def lagrange_para_check(func, constraints, options, tol):
+    """check the parameter of lagrange method."""
+
+    if not callable(func):
+        raise TypeError("'func' must be of type function, but got {}".format(type(func)))
+    if 'coincide_func' in options:
+        if not callable(options['coincide_fun']):
+            raise TypeError("'coincide_fun' must be of type function, but got {}".format(type(options['coincide_fun'])))
+    for constraint in constraints:
+        if not callable(constraint):
+            raise TypeError("'constraint' must be of type function, but got {}".format(type(constraint)))
+    if tol is not None:
+        if len(tol) != len(constraints):
+            raise ValueError("The len of tol must be same as the len of constraints")
+
+
 def minimize(func, x0, args=(), method=None, jac=None, hess=None, hessp=None, bounds=None, constraints=(),
              tol=None, callback=None, options=None):
     r"""Minimization of scalar function of one or more variables.
@@ -66,6 +83,7 @@ def minimize(func, x0, args=(), method=None, jac=None, hess=None, hessp=None, bo
           multi-dimensional Tensor, but support for both is planned.
 
         - `minimize` is not supported on Windows platform yet.
+        -  `LAGRANGE` method is only supported on "GPU".
 
     Args:
         func (Callable): the objective function to be minimized, :math:`fun(x, *args) -> float`,
@@ -75,14 +93,16 @@ def minimize(func, x0, args=(), method=None, jac=None, hess=None, hessp=None, bo
         x0 (Tensor): initial guess. Array of real elements of size :math:`(n,)`, where `n` is
             the number of independent variables.
         args (Tuple): extra arguments passed to the objective function. Default: ().
-        method (str): solver type. Should be one of `"BFGS"` and `"LBFGS"`.
+        method (str): solver type. Should be one of `"BFGS"` and `"LBFGS"`, `"LAGRANGE"`.
         jac (Callable, optional): method for computing the gradient vector. Only for `"BFGS"` and `"LBFGS"`.
             if it is None, the gradient will be estimated with gradient of ``func``.
             if it is a callable, it should be a function that returns the gradient vector:
             :math:`jac(x, *args) -> array\_like, shape (n,)`
-            where x is an array with shape (n,) and args is a tuple with the fixed parameters.
+            where x is an array with shape :math:`(n,)` and args is a tuple with the fixed parameters.
         tol (float, optional): tolerance for termination. For detailed control, use solver-specific
             options. Default: None.
+        constraints(Callable, optional): representing the inequality constrains, each function in constrains indicates
+            the function < 0 as an inequality constrain.
         options (Mapping[str, Any], optional): a dictionary of solver options. All methods accept the following
             generic options, Default: None.
 
@@ -91,6 +111,19 @@ def minimize(func, x0, args=(), method=None, jac=None, hess=None, hessp=None, bo
             - maxiter (int): Maximum number of iterations to perform. Depending on the
               method each iteration may use several function evaluations.
 
+        The follow options are exclusive to Lagrange method:
+
+            - save_tol (list): list of saving tolerance, with the same length with 'constrains'.
+            - obj_weight (float): weight for objective function, usually between 1.0 - 100000.0.
+            - lower (Tensor): lower bound constrain for variables, must have same shape with x0.
+            - upper (Tensor): upper bound constrain for variables, must have same shape with x0.
+            - learning_rate (float): learning rate for each Adam step.
+            - coincide_func (Callable): sub-function representing the common parts between objective function
+              and constrains to avoid redundant computation.
+            - rounds (int): times to update Lagrange multipliers.
+            - steps (int): steps to apply Adam per round.
+            - log_sw (bool): whether to print the loss at each step.
+
     Returns:
         OptimizeResults, object holding optimization results.
 
@@ -154,4 +187,10 @@ def minimize(func, x0, args=(), method=None, jac=None, hess=None, hessp=None, bo
                                njev=results.ngev,
                                nit=results.k)
 
+
+    if method.lower() == 'lagrange':
+        lagrange_para_check(func, constraints, options, tol)
+        results = minimize_lagrange(func, x0, constraints, tol, **options)
+        return results
+
     raise ValueError("Method {} not recognized".format(method))

mindspore/scipy/sparse/__init__.py

@@ -14,6 +14,6 @@
 # ============================================================================
 """Sparse submodule"""
 from . import linalg
-from .linalg import cg, gmres, bicgstab
+from .linalg import cg
 
-__all__ = ["cg", "gmres", "bicgstab"]
+__all__ = ["cg"]