DFO-LS 1.2.1__py3-none-any.whl → 1.5.0__py3-none-any.whl

dfols/trust_region.py

@@ -11,6 +11,15 @@ produces a new vector d which (approximately) solves the trust region subproblem
 The other outputs: gnew is the gradient of the model at d, and crvmin has
 information about the curvature of the model at the solution.
 
+For handling arbitrary constraints, the call is
+    d, gnew, crvmin = ctrsbox(xopt, g, H, projections, delta)
+which produces a new vector d approximately solving the constrained trust region subproblem:
+    min_{d}  g'*d + 0.5*d'*H*d
+    s.t.    ||d|| <= delta
+            xopt + d is feasible w.r.t. the constraint set C
+The other outputs: gnew is the gradient of the model at d, and crvmin has
+information about the curvature of the model at the solution.
+
 We also provide a function for maximising the absolute value of a linear function
 inside a similar trust region - this is useful for geometry steps.
 The call
@@ -20,9 +29,16 @@ solves
     s.t.  lower <= x <= upper
           ||x-xbase|| <= Delta
 With this value, the variable d=x-xbase solves the problem
-    min_s  abs(c + g' * d)
+    min_d  abs(c + g' * d)
     s.t.   lower <= xbase + d <= upper
           ||d|| <= delta
+Again, we have a version of this for handling arbitrary constraints
+The call
+    x = ctrsbox_geometry(xbase, c, g, projections, Delta)
+Solves
+    min_d  abs(c + g' * d)
+    s.t.   xbase + d is feasible w.r.t. the constraint set C
+          ||d|| <= delta
 
 Notes
 ----
@@ -54,7 +70,7 @@ alternative licensing.
 # Ensure compatibility with Python 2
 from __future__ import absolute_import, division, print_function, unicode_literals
 
-from math import sqrt
+from math import sqrt, ceil
 import numpy as np
 try:
     import trustregion
@@ -63,14 +79,157 @@ except ImportError:
     # Fall back to Python implementation
     USE_FORTRAN = False
 
+from .util import dykstra, pball, pbox, sumsq, model_value, remove_scaling
 
-from .util import sumsq
+__all__ = ['ctrsbox_sfista', 'ctrsbox_pgd', 'ctrsbox_geometry', 'trsbox', 'trsbox_geometry']
 
+ZERO_THRESH = 1e-14
 
-__all__ = ['trsbox', 'trsbox_geometry']
+def ctrsbox_sfista(xopt, g, H, projections, delta, h, L_h, prox_uh, argsh=(), argsprox=(), func_tol=1e-3, max_iters=500, d_max_iters=100, d_tol=1e-10, use_fortran=USE_FORTRAN, scaling_changes=None, sfista_iters_scale=1.0):
+    n = xopt.size
+    assert xopt.shape == (n,), "xopt has wrong shape (should be vector)"
+    assert g.shape == (n,), "g and xopt have incompatible sizes"
+    assert len(H.shape) == 2, "H must be a matrix"
+    assert H.shape == (n,n), "H and xopt have incompatible sizes"
+    assert np.allclose(H, H.T), "H must be symmetric"
+    assert delta > 0.0, "delta must be strictly positive"
 
-ZERO_THRESH = 1e-14
+    # Initialization
+    d = np.zeros(n) # start with zero vector
+    y = np.zeros(n)
+    t = 1
+    k_H = np.linalg.norm(H, 2)
+    crvmin = -1.0
+    
+    # Number of iterations & smoothing parameter, from Theorem 10.57 in 
+    #   [A. Beck. First-order methods in optimization, SIAM, 2017]
+    # We do not use the values of k and mu given in the theorem statement, but rather the intermediate
+    # results on p313 (K1 for number of iterations, and the immediate next line for mu)
+    # Note: in the book's notation, Gamma=delta^2, alpha=1, beta=L_h^2/2, Lf=k_H [alpha and beta from Thm 10.51]
+    try:
+        MAX_LOOP_ITERS = ceil(sfista_iters_scale * delta * (L_h+sqrt(L_h*L_h+2*k_H*func_tol)) / func_tol)
+        MAX_LOOP_ITERS = min(MAX_LOOP_ITERS, max_iters)
+    except ValueError:
+        MAX_LOOP_ITERS = max_iters
+    u =  2 * delta / (MAX_LOOP_ITERS * L_h) # smoothing parameter
+    # u = 2 * func_tol / (L_h ** 2 + L_h * sqrt(L_h ** 2 + 2 * k_H * func_tol))  # the above choice works better in practice
+
+    def gradient_Fu(xopt, g, H, u, prox_uh, d):
+    # Calculate gradient_Fu,
+    # where Fu(d) := g(d) + h_u(d) and h_u(d) is a 1/u-smooth approximation of h.
+    # We assume that h is globally Lipschitz continous with constant L_h,
+    # then we can let h_u(d) be the Moreau Envelope M_h_u(d) of h.
+        return g + H @ d + (xopt + d - prox_uh(remove_scaling(xopt + d, scaling_changes), u, *argsprox)) / u
+
+    # Lipschitz constant of gradient_Fu
+    l = k_H + 1 / u 
+
+    # trust region is a ball of radius delta around xopt
+    trproj = lambda w: pball(w, xopt, delta)
+
+    # combine trust region constraints with user-entered constraints
+    P = list(projections)  # make a copy of the projections list
+    P.append(trproj)
+    def proj(d0):
+        p = dykstra(P, xopt+d0, max_iter=d_max_iters, tol=d_tol)
+        # we want the step only, so we subtract xopt
+        # from the new point: proj(xk+d) - xk
+        return p - xopt
+
+    # general step
+    model_value_best = model_value(g, H, d, xopt, h, argsh, scaling_changes)
+    d_best = d.copy()
+    for k in range(MAX_LOOP_ITERS):
+        prev_d = d.copy()
+        prev_t = t
+        # gradient_Fu at y
+        g_Fu = gradient_Fu(xopt, g, H, u, prox_uh, d, *argsprox)
+
+        # main update step
+        d = proj(y - g_Fu / l)
+        new_model_value = model_value(g, H, d, xopt, h, argsh, scaling_changes)
+        if new_model_value < model_value_best:
+            d_best = d.copy()
+            model_value_best = new_model_value
+
+        # update true gradient
+        # gnew is the gradient of the smoothed function
+        gnew = gradient_Fu(xopt, g, H, u, prox_uh, d, *argsprox)
+
+        # update CRVMIN
+        crv = d.dot(H).dot(d)/sumsq(d) if sumsq(d) >= ZERO_THRESH else crvmin
+        crvmin = min(crvmin, crv) if crvmin != -1.0 else crv
+        
+        # momentum update
+        t = (1 + sqrt(1 + 4*t*t)) / 2
+        y = d + (prev_t - 1) * (d - prev_d) / t
+    return d, gnew, crvmin
+
+def ctrsbox_pgd(xopt, g, H, projections, delta, d_max_iters=100, d_tol=1e-10, use_fortran=USE_FORTRAN):
+    n = xopt.size
+    assert xopt.shape == (n,), "xopt has wrong shape (should be vector)"
+    assert g.shape == (n,), "g and xopt have incompatible sizes"
+    assert len(H.shape) == 2, "H must be a matrix"
+    assert H.shape == (n,n), "H and xopt have incompatible sizes"
+    assert np.allclose(H, H.T), "H must be symmetric"
+    assert delta > 0.0, "delta must be strictly positive"
+
+    d = np.zeros((n,))
+    gnew = g.copy()
+    gy = g.copy()
+    crvmin = -1.0
+    y = d.copy()
+    eta = 1.2 # L backtrack scaling factor
+    t = 1
+
+    # Initial guess of L is norm(Hessian)
+    L = np.linalg.norm(H, 2)
+
+    # trust region is a ball of radius delta around xopt
+    trproj = lambda w: pball(w, xopt, delta)
+
+    # combine trust region constraints with user-entered constraints
+    P = list(projections)  # make a copy of the projections list
+    P.append(trproj)
+    def proj(d0):
+        p = dykstra(P, xopt+d0, max_iter=d_max_iters, tol=d_tol)
+        # we want the step only, so we subtract xopt
+        # from the new point: proj(xk+d) - xk
+        return p - xopt
+
+    MAX_LOOP_ITERS = 100 * n ** 2
+
+    # projected GD loop 
+    for ii in range(MAX_LOOP_ITERS):
+        w = y - (1/L)*gy
+        prev_d = d.copy()
+        d = proj(w)
+
+        # size of step taken
+        s = d - prev_d
+        stplen = np.linalg.norm(s)
 
+        # update true gradient
+        gnew += H.dot(s)
+
+        # update CRVMIN
+        crv = s.dot(H).dot(s)/sumsq(s) if sumsq(s) >= ZERO_THRESH else crvmin
+        crvmin = min(crvmin, crv) if crvmin != -1.0 else crv
+
+        # exit condition
+        if stplen <= ZERO_THRESH:
+            break
+
+        # momentum update
+        prev_t = t
+        t = (1 + np.sqrt(1 + 4 * t ** 2))/2
+        prev_y = y.copy()
+        y = d + s*(prev_t - 1)/t
+
+        # update gradient w.r.t y
+        gy += H.dot(y - prev_y)
+
+    return d, gnew, crvmin
 
 def trsbox(xopt, g, H, sl, su, delta, use_fortran=USE_FORTRAN):
     if use_fortran:
@@ -103,7 +262,7 @@ def trsbox(xopt, g, H, sl, su, delta, use_fortran=USE_FORTRAN):
     iterc = 0
     nact = 0  # number of fixed variables
 
-    xbdi = np.zeros((n,), dtype=np.int)  # fix x_i at bounds? [values -1, 0, 1]
+    xbdi = np.zeros((n,), dtype=int)  # fix x_i at bounds? [values -1, 0, 1]
     xbdi[(xopt <= sl) & (g >= 0.0)] = -1
     xbdi[(xopt >= su) & (g <= 0.0)] = 1
 
@@ -405,8 +564,63 @@ def ball_step(x0, g, Delta):
     if sqrt(gsqnorm) < ZERO_THRESH:  # Error catching: if g=0, make no step
         return 0.0
     else:
-        return (sqrt(gdotx0**2 + gsqnorm*(Delta**2 - x0sqnorm)) - gdotx0) / gsqnorm
+        # Sqrt had negative input on prob 46 in OG DFOLS with noise
+        #  print("Inside of the sqrt:", gdotx0**2 + gsqnorm*(Delta**2 - x0sqnorm))
+        # Got Inside of the sqrt: -3.608971127647144e-42
+        # Added max(0,...) here
+        return (sqrt(np.maximum(0,gdotx0**2 + gsqnorm*(Delta**2 - x0sqnorm))) - gdotx0) / gsqnorm
+
+def ctrsbox_linear(xbase, g, projections, Delta, d_max_iters=100, d_tol=1e-10, use_fortran=USE_FORTRAN):
+    # Solve the convex program:
+    #   min_d   g' * d
+    #   s.t.    xbase + d is feasible w.r.t. constraint set C
+    #           ||d||^2 <= Delta^2
+
+    n = g.size
+    d = np.zeros((n,))
+    y = d.copy()
+    t = 1
+    dirn = -g
+    cons_dirns = []
+
+    # If g[i] = 0, never step along this direction
+    constant_directions = np.where(np.abs(dirn) < ZERO_THRESH)[0]
+    dirn[constant_directions] = 0.0
+
+    # trust region is a ball of radius delta centered around xbase
+    trproj = lambda w: pball(w, xbase, Delta)
 
+    # combine trust region constraints with user-entered constraints
+    P = list(projections)  # make a copy of the projections list
+    P.append(trproj)
+    def proj(d0):
+        p = dykstra(P, xbase + d0, max_iter=d_max_iters, tol=d_tol)
+        # we want the step only, so we subtract
+        # xbase from the new point: proj(xk + d) - xk
+        return p - xbase
+
+    MAX_LOOP_ITERS = 100 * n ** 2
+
+    # projected GD loop 
+    for ii in range(MAX_LOOP_ITERS):
+        w = y + dirn
+        prev_d = d.copy()
+        d = proj(w)
+
+        s = d - prev_d
+        stplen = np.linalg.norm(s)
+
+        # exit condition
+        if stplen <= ZERO_THRESH:
+            break
+
+        # 'momentum' update
+        prev_t = t
+        t = (1 + np.sqrt(1 + 4 * t ** 2))/2
+        prev_y = y.copy()
+        y = d + s*(prev_t - 1)/t
+
+    return d
 
 def trsbox_linear(g, a_in, b_in, Delta, use_fortran=USE_FORTRAN):
     # Solve the convex program:
@@ -466,6 +680,22 @@ def trsbox_linear(g, a_in, b_in, Delta, use_fortran=USE_FORTRAN):
             dirn[idx_hit] = 0.0  # no more searching this direction
     return x
 
+def ctrsbox_geometry(xbase, c, g, projections, Delta, d_max_iters=100, d_tol=1e-10, use_fortran=USE_FORTRAN):
+    # Given a Lagrange polynomial defined by: L(x) = c + g' * (x - xbase)
+    # Maximise |L(x)| in a box + trust region - that is, solve:
+    #   max_x  abs(c + g' * (x - xbase))
+    #    s.t.  x is feasible w.r.t constraint set C
+    #          ||x-xbase|| <= Delta
+    # Setting s = x-xbase (or x = xbase + s), this is equivalent to:
+    #   max_s  abs(c + g' * s)
+    #   s.t.   xbase + s is is feasible w.r.t constraint set C
+    #          ||s|| <= Delta
+    smin = ctrsbox_linear(xbase, g, projections, Delta, d_max_iters=100, d_tol=1e-10, use_fortran=use_fortran)  # minimise g' * s
+    smax = ctrsbox_linear(xbase, -g, projections, Delta, d_max_iters=100, d_tol=1e-10, use_fortran=use_fortran)  # maximise g' * s
+    if abs(c + np.dot(g, smin)) >= abs(c + np.dot(g, smax)):  # choose the one with largest absolute value
+        return smin
+    else:
+        return smax
 
 def trsbox_geometry(xbase, c, g, lower, upper, Delta, use_fortran=USE_FORTRAN):
     # Given a Lagrange polynomial defined by: L(x) = c + g' * (x - xbase)

dfols/util.py

@@ -27,11 +27,14 @@ from __future__ import absolute_import, division, print_function, unicode_litera
 
 import logging
 import numpy as np
+import scipy.linalg as LA
 import sys
 
 
-__all__ = ['sumsq', 'eval_least_squares_objective', 'model_value', 'random_orthog_directions_within_bounds',
-           'random_directions_within_bounds', 'apply_scaling', 'remove_scaling']
+__all__ = ['sumsq', 'eval_least_squares_with_regularisation', 'model_value', 'random_orthog_directions_within_bounds',
+           'random_directions_within_bounds', 'apply_scaling', 'remove_scaling', 'pbox', 'pball', 'dykstra', 'qr_rank']
+
+module_logger = logging.getLogger(__name__) 
 
 
 def sumsq(x):
@@ -44,9 +47,9 @@ def sumsq(x):
     return np.dot(x, x)
 
 
-def eval_least_squares_objective(objfun, x, args=(), verbose=True, eval_num=0, pt_num=0, full_x_thresh=6, check_for_overflow=True):
+def eval_least_squares_with_regularisation(objfun, x, h=None, argsf=(), argsh=(), verbose=True, eval_num=0, pt_num=0, full_x_thresh=6, check_for_overflow=True):
     # Evaluate least squares function
-    fvec = objfun(x, *args)
+    fvec = objfun(x, *argsf)
 
     if check_for_overflow:
         try:
@@ -59,20 +62,31 @@ def eval_least_squares_objective(objfun, x, args=(), verbose=True, eval_num=0, p
     else:
         f = sumsq(fvec)
 
+    # objective = least-squares + regularisation
+    obj = f
+    if h is not None:
+        # Evaluate regularisation term
+        hvalue = h(x, *argsh)
+        obj = f + hvalue
+
     if verbose:
         if len(x) < full_x_thresh:
-            logging.info("Function eval %i at point %i has f = %.15g at x = " % (eval_num, pt_num, f) + str(x))
+            module_logger.info("Function eval %i at point %i has obj = %.15g at x = " % (eval_num, pt_num, obj) + str(x))
         else:
-            logging.info("Function eval %i at point %i has f = %.15g at x = [...]" % (eval_num, pt_num, f))
+            module_logger.info("Function eval %i at point %i has obj = %.15g at x = [...]" % (eval_num, pt_num, obj))
 
-    return fvec, f
+    return fvec, obj
 
 
-def model_value(g, H, s):
-    # Calculate model value (s^T * g + 0.5* s^T * H * s) = s^T * (gopt + 0.5 * H*s)
+def model_value(g, H, s, xopt=(), h=None,argsh=(), scaling_changes=None):
+    # Calculate model value (s^T * g + 0.5* s^T * H * s) + h(xopt + s) = s^T * (gopt + 0.5 * H*s) + h(xopt + s)
     assert g.shape == s.shape, "g and s have incompatible sizes"
     Hs = H.dot(s)
-    return np.dot(s, g + 0.5*Hs)
+    rtn = np.dot(s, g + 0.5*Hs)
+    if h is not None:
+        hvalue = h(remove_scaling(xopt+s, scaling_changes), *argsh)
+        rtn += hvalue
+    return rtn
 
 
 def get_scale(dirn, delta, lower, upper):
@@ -207,3 +221,50 @@ def remove_scaling(x_scaled, scaling_changes):
     shift, scale = scaling_changes
     return shift + x_scaled * scale
 
+
+def dykstra(P,x0,max_iter=100,tol=1e-10):
+    x = x0.copy()
+    p = len(P)
+    y = np.zeros((p,x0.shape[0]))
+
+    n = 0
+    cI = float('inf')
+    while n < max_iter and cI >= tol:
+        cI = 0
+        for i in range(0,p):
+            # Update iterate
+            prev_x = x.copy()
+            x = P[i](prev_x - y[i,:])
+
+            # Update increment
+            prev_y = y[i,:].copy()
+            y[i,:] = x - (prev_x - prev_y)
+
+            # Stop condition
+            cI += np.linalg.norm(prev_y - y[i,:])**2
+
+        n += 1
+
+    return x
+
+
+def pball(x,c,r):
+    return c + (r/np.max([np.linalg.norm(x-c),r]))*(x-c)
+
+
+def pbox(x,l,u):
+    return np.minimum(np.maximum(x,l), u)
+
+'''
+Calculates rank of square matrix with QR.
+We use the fact that the rank of a square matrix A
+can be given by the number of nonzero diagonal elements of
+R in the QR factorization of A.
+'''
+def qr_rank(A,tol=1e-15):
+    m,n = A.shape
+    assert m == n, "Input matrix must be square"
+    Q,R = LA.qr(A)
+    D = np.abs(np.diag(R))
+    rank = np.sum(D > tol) 
+    return rank, D

DFO_LS-1.2.1.dist-info/RECORD

@@ -1,16 +0,0 @@
-dfols/__init__.py,sha256=WdWULFDmitwdbZ3JEcrbkb5DjXr0ZM6efAqSe-iPsr8,1620
-dfols/controller.py,sha256=7puR1_zuu47Vr-xEhqubEJRzLv2Yja3X2O_WeDaDSV0,35652
-dfols/diagnostic_info.py,sha256=2kEUkL-MS4eDENUf1r2hOWsntP8OxMDKi_kyHmrC9V4,6081
-dfols/hessian.py,sha256=DKOm520rrFFaXVXu-BsD7aj2_ilJohxZ2tnMsnSa8zE,4265
-dfols/model.py,sha256=OHi-Y50OajpIauzfGcYPdCSrM8gvXYTGjb3NY7enk9s,18020
-dfols/params.py,sha256=1Mplj96q9anYa6d3PCkXgsxp-tc9fP4mNtQxwm7bveE,16749
-dfols/solver.py,sha256=RPUQpgPRkcUVMvc6tJ-0bYckg8EesMMeQ1meRJk3u_k,59180
-dfols/trust_region.py,sha256=75Z0z2d6ENh8t9t_e95IVwgNaWlZpyZF4rdkMJ54qhc,19152
-dfols/util.py,sha256=gpDa71PSFhFMqIJdmndnJ0vE25BiSulscRPS6MlIyaE,8667
-dfols/version.py,sha256=GQUgqQ6kgZyC6ohszFAofexRKp-dKwWY81bo7q2hlI8,926
-DFO_LS-1.2.1.dist-info/LICENSE.txt,sha256=jOtLnuWt7d5Hsx6XXB2QxzrSe2sWWh3NgMfFRetluQM,35147
-DFO_LS-1.2.1.dist-info/METADATA,sha256=nyESs24W-UXP0yYFZ8XX7YGOv39pIPKc8UAvRHrLfEc,7039
-DFO_LS-1.2.1.dist-info/WHEEL,sha256=g4nMs7d-Xl9-xC9XovUrsDHGXt-FT0E17Yqo92DEfvY,92
-DFO_LS-1.2.1.dist-info/top_level.txt,sha256=UfxRhaDN8HQx2_l17KbrDrERJ90OCN7VKkDMpYYbRLU,6
-DFO_LS-1.2.1.dist-info/zip-safe,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
-DFO_LS-1.2.1.dist-info/RECORD,,

DFO_LS-1.2.1.dist-info/zip-safe

@@ -1 +0,0 @@
-

dfols/version.py

@@ -1,25 +0,0 @@
-"""
-Version number
-====
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License
-along with this program. If not, see <http://www.gnu.org/licenses/>.
-
-The development of this software was sponsored by NAG Ltd. (http://www.nag.co.uk)
-and the EPSRC Centre For Doctoral Training in Industrially Focused Mathematical
-Modelling (EP/L015803/1) at the University of Oxford. Please contact NAG for
-alternative licensing.
-
-"""
-
-__version__ = '1.2.1'