DFO-LS 1.4.1__py3-none-any.whl → 1.5.1__py3-none-any.whl

dfols/model.py

@@ -36,7 +36,7 @@ import numpy as np
 import scipy.linalg as LA
 
 from .trust_region import trsbox_geometry
-from .util import sumsq, dykstra
+from .util import sumsq, dykstra, remove_scaling
 
 __all__ = ['Model']
 
@@ -44,8 +44,8 @@ module_logger = logging.getLogger(__name__)
 
 
 class Model(object):
-    def __init__(self, npt, x0, r0, xl, xu, projections, r0_nsamples, n=None, m=None, abs_tol=1e-12, rel_tol=1e-20, precondition=True,
-                 do_logging=True):
+    def __init__(self, npt, x0, r0, xl, xu, projections, r0_nsamples, h=None, argsh=(), n=None, m=None, abs_tol=1e-12, rel_tol=1e-20, precondition=True,
+                 do_logging=True, scaling_changes=None):
         if n is None:
             n = len(x0)
         if m is None:
@@ -56,11 +56,15 @@ class Model(object):
         assert xu.shape == (n,), "xu has wrong shape (got %s, expect (%g,))" % (str(xu.shape), n)
         assert r0.shape == (m,), "r0 has wrong shape (got %s, expect (%g,))" % (str(r0.shape), m)
         self.do_logging = do_logging
+        self.scaling_changes = scaling_changes
         self.dim = n
         self.resid_dim = m
         self.num_pts = npt
         self.npt_so_far = 1  # number of points added so far (with function values)
 
+        self.h = h
+        self.argsh = argsh
+
         # Initialise to blank some useful stuff
         # Interpolation points
         self.xbase = x0.copy()
@@ -72,12 +76,17 @@ class Model(object):
         # Function values
         self.fval_v = np.inf * np.ones((npt, m))  # residuals for each xpt
         self.fval_v[0, :] = r0.copy()
-        self.fval = np.inf * np.ones((npt, ))  # overall objective value for each xpt
-        self.fval[0] = sumsq(r0)
+        
+        self.objval = np.inf * np.ones((npt, ))  # overall objective value for each xpt
+        self.objval[0] = sumsq(r0)
+        if h is not None:
+            self.objval[0] += h(remove_scaling(x0, self.scaling_changes), *argsh)
         self.kopt = 0  # index of current iterate (should be best value so far)
         self.nsamples = np.zeros((npt,), dtype=int)  # number of samples used to evaluate objective at each point
         self.nsamples[0] = r0_nsamples
-        self.fbeg = self.fval[0]  # f(x0), saved to check for sufficient reduction
+        self.objbeg = self.objval[0]  # f(x0), saved to check for sufficient reduction
+        self.eval_num = np.zeros((npt,), dtype=int)  # which evaluation number (1-indexed, nx not nf) is currently stored self.points[k,:]
+        self.eval_num[0] = 1
 
         # Termination criteria
         self.abs_tol = abs_tol
@@ -86,13 +95,16 @@ class Model(object):
         # Model information
         self.model_const = np.zeros((m, ))  # constant term for model m(s) = c + J*s
         self.model_jac = np.zeros((m, n))  # Jacobian term for model m(s) = c + J*s
+        self.model_jac_eval_nums = None  # which evaluation numbers (1-indexed, nx not nf) were used to build model_jac
 
         # Saved point (in absolute coordinates) - always check this value before quitting solver
         self.xsave = None
         self.rsave = None
-        self.fsave = None
+        self.objsave = None
         self.jacsave = None
         self.nsamples_save = None
+        self.eval_num_save = None
+        self.jacsave_eval_nums = None
 
         # Factorisation of interpolation matrix
         self.factorisation_current = False
@@ -118,8 +130,8 @@ class Model(object):
     def ropt(self):
         return self.fval_v[self.kopt, :]  # residuals for current iterate
 
-    def fopt(self):
-        return self.fval[self.kopt]
+    def objopt(self):
+        return self.objval[self.kopt]
 
     def xpt(self, k, abs_coordinates=False):
         assert 0 <= k < self.npt(), "Invalid index %g" % k
@@ -135,9 +147,9 @@ class Model(object):
         assert 0 <= k < self.npt(), "Invalid index %g" % k
         return self.fval_v[k, :]
 
-    def fval(self, k):
+    def objval(self, k):
         assert 0 <= k < self.npt(), "Invalid index %g" % k
-        return self.fval[k]
+        return self.objval[k]
 
     def as_absolute_coordinates(self, x, full_dykstra=False):
         # If x were an interpolation point, get the absolute coordinates of x
@@ -167,7 +179,7 @@ class Model(object):
             sq_distances[k] = sumsq(self.points[k, :] - xopt)
         return sq_distances
 
-    def change_point(self, k, x, rvec, allow_kopt_update=True):
+    def change_point(self, k, x, rvec, eval_num, allow_kopt_update=True):
         # Update point k to x (w.r.t. xbase), with residual values fvec
         if k >= self.npt_so_far and self.npt_so_far < self.num_pts:
             assert k == self.npt_so_far, "Growing: updating wrong point"
@@ -177,18 +189,22 @@ class Model(object):
 
         self.points[k, :] = x.copy()
         self.fval_v[k, :] = rvec.copy()
-        self.fval[k] = sumsq(rvec)
+        self.objval[k] = sumsq(rvec)
+        if self.h is not None:
+            self.objval[k] += self.h(remove_scaling(self.xbase + x, self.scaling_changes), *self.argsh)
         self.nsamples[k] = 1
+        self.eval_num[k] = eval_num
         self.factorisation_current = False
 
-        if allow_kopt_update and self.fval[k] < self.fopt():
+        if allow_kopt_update and self.objval[k] < self.objopt():
             self.kopt = k
         return
 
     def swap_points(self, k1, k2):
         self.points[[k1, k2], :] = self.points[[k2, k1], :]
         self.fval_v[[k1, k2], :] = self.fval_v[[k2, k1], :]
-        self.fval[[k1, k2]] = self.fval[[k2, k1]]
+        self.objval[[k1, k2]] = self.objval[[k2, k1]]
+        self.eval_num[[k1, k2]] = self.eval_num[[k2, k1]]
         if self.kopt == k1:
             self.kopt = k2
         elif self.kopt == k2:
@@ -201,22 +217,28 @@ class Model(object):
         assert 0 <= k < self.npt(), "Invalid index %g" % k
         t = float(self.nsamples[k]) / float(self.nsamples[k] + 1)
         self.fval_v[k, :] = t * self.fval_v[k, :] + (1 - t) * rvec_extra
-        self.fval[k] = sumsq(self.fval_v[k, :])
+        # NOTE: how to sample when we have h? still at xpt(k), then add h(xpt(k)). Modify test if incorrect!
+        self.objval[k] = sumsq(self.fval_v[k, :])
+        if self.h is not None:
+            self.objval[k] += self.h(remove_scaling(self.xbase + self.points[k, :], self.scaling_changes), *self.argsh)
         self.nsamples[k] += 1
 
-        self.kopt = np.argmin(self.fval[:self.npt()])  # make sure kopt is always the best value we have
+        self.kopt = np.argmin(self.objval[:self.npt()])  # make sure kopt is always the best value we have
         return
 
-    def add_new_point(self, x, rvec):
+    def add_new_point(self, x, rvec, eval_num):
         self.points = np.append(self.points, x.reshape((1, self.n())), axis=0)  # append row to xpt
         self.fval_v = np.append(self.fval_v, rvec.reshape((1, self.m())), axis=0)  # append row to fval_v
-        f = np.dot(rvec, rvec)
-        self.fval = np.append(self.fval, f)  # append entry to fval
+        obj = sumsq(rvec)
+        if self.h is not None:
+            obj += self.h(remove_scaling(self.xbase + x, self.scaling_changes), *self.argsh)
+        self.objval = np.append(self.objval, obj)  # append entry to fval
         self.nsamples = np.append(self.nsamples, 1)  # add new sample number
+        self.eval_num = np.append(self.eval_num, eval_num)  # add new evaluation number
         self.num_pts += 1  # make sure npt is updated
         self.npt_so_far += 1
 
-        if f < self.fopt():
+        if obj < self.objopt():
             self.kopt = self.npt() - 1
 
         self.factorisation_current = False
@@ -235,28 +257,33 @@ class Model(object):
         self.model_const += np.dot(self.model_jac, xbase_shift)
         return
 
-    def save_point(self, x, rvec, nsamples, x_in_abs_coords=True):
-        f = sumsq(rvec)
-        if self.fsave is None or f <= self.fsave:
-            self.xsave = x.copy() if x_in_abs_coords else self.as_absolute_coordinates(x)
+    def save_point(self, x, rvec, nsamples, eval_num, x_in_abs_coords=True):
+        xabs = x.copy() if x_in_abs_coords else self.as_absolute_coordinates(x)
+        obj = sumsq(rvec)
+        if self.h is not None:
+            obj += self.h(remove_scaling(xabs, self.scaling_changes), *self.argsh)
+        if self.objsave is None or obj <= self.objsave:
+            self.xsave = xabs
             self.rsave = rvec.copy()
-            self.fsave = f
-            self.jacsave = self.model_jac.copy()
+            self.objsave = obj
+            self.jacsave = self.model_jac.copy() if self.model_jac is not None else None
             self.nsamples_save = nsamples
+            self.eval_num_save = eval_num
+            self.jacsave_eval_nums = self.model_jac_eval_nums.copy() if self.model_jac_eval_nums is not None else None
             return True
         else:
             return False  # this value is worse than what we have already - didn't save
 
     def get_final_results(self):
-        # Return x and fval for optimal point (either from xsave+fsave or kopt)
-        if self.fsave is None or self.fopt() <= self.fsave:  # optimal has changed since xsave+fsave were last set
-            return self.xopt(abs_coordinates=True).copy(), self.ropt().copy(), self.fopt(), self.model_jac.copy(), self.nsamples[self.kopt]
+        # Return x and objval for optimal point (either from xsave+objsave or kopt)
+        if self.objsave is None or self.objopt() <= self.objsave:  # optimal has changed since xsave+objsave were last set
+            return self.xopt(abs_coordinates=True).copy(), self.ropt().copy(), self.objopt(), self.model_jac.copy(), self.nsamples[self.kopt], self.eval_num[self.kopt], self.model_jac_eval_nums
         else:
-            return self.xsave.copy(), self.rsave.copy(), self.fsave, self.jacsave, self.nsamples_save
+            return self.xsave.copy(), self.rsave.copy(), self.objsave, self.jacsave, self.nsamples_save, self.eval_num_save, self.jacsave_eval_nums
 
     def min_objective_value(self):
         # Get termination criterion for f small: f <= abs_tol or f <= rel_tol * f0
-        return max(self.abs_tol, self.rel_tol * self.fbeg)
+        return max(self.abs_tol, self.rel_tol * self.objbeg)
 
     def model_value(self, d, d_based_at_xopt=True, with_const_term=False):
         if d_based_at_xopt:
@@ -350,6 +377,7 @@ class Model(object):
         J_old = self.model_jac.copy()
         self.model_jac = dg[1:,:].T
         self.model_const = dg[0,:] - np.dot(self.model_jac, xopt)  # shift base to xbase
+        self.model_jac_eval_nums = self.eval_num.copy()
         if verbose or get_chg_J:
             norm_J_error = np.linalg.norm(self.model_jac - J_old, ord='fro')**2
             linalg_resid = np.linalg.norm(W.dot(dg) - rhs)**2
@@ -375,7 +403,7 @@ class Model(object):
         return True, interp_error, sqrt(norm_J_error), linalg_resid, ls_interp_cond_num  # flag ok
 
     def build_full_model(self):
-        # Build full least squares objective model from mini-models
+        # Build full least squares model from mini-models
         # Centred around xopt
         r = self.model_const + np.dot(self.model_jac, self.xopt())  # constant term (for inexact interpolation)
         J = self.model_jac

dfols/params.py

@@ -82,7 +82,7 @@ class ParameterList(object):
         self.params["restarts.use_soft_restarts"] = True
         self.params["restarts.soft.num_geom_steps"] = 3
         self.params["restarts.soft.move_xk"] = True
-        self.params["restarts.soft.max_fake_successful_steps"] = maxfun  # number ratio>0 steps below fsave allowed
+        self.params["restarts.soft.max_fake_successful_steps"] = maxfun  # number ratio>0 steps below objsave allowed
         self.params["restarts.hard.use_old_rk"] = True  # recycle r(xk) from previous run?
         self.params["restarts.increase_npt"] = False
         self.params["restarts.increase_npt_amt"] = 1
@@ -109,12 +109,20 @@ class ParameterList(object):
         self.params["growing.full_rank.min_sing_val"] = 1e-6  # absolute floor on singular values
         self.params["growing.full_rank.svd_max_jac_cond"] = 1e8  # maximum condition number of Jacobian
         self.params["growing.perturb_trust_region_step"] = False  # add random direction onto TRS solution?
+        
         # Dykstra's algorithm
         self.params["dykstra.d_tol"] = 1e-10
         self.params["dykstra.max_iters"] = 100
+        
         # Matrix rank algorithm
         self.params["matrix_rank.r_tol"] = 1e-18
-
+        
+        # Function tolerance when applying S-FISTA method
+        self.params["func_tol.criticality_measure"] = 1e-3
+        self.params["func_tol.tr_step"] = 1-1e-1
+        self.params["func_tol.max_iters"] = 500
+        self.params["sfista.max_iters_scaling"] = 2.0
+        
         self.params_changed = {}
         for p in self.params:
             self.params_changed[p] = False
@@ -268,6 +276,14 @@ class ParameterList(object):
             type_str, nonetype_ok, lower, upper = 'int', False, 0, None
         elif key == "matrix_rank.r_tol":
             type_str, nonetype_ok, lower, upper = 'float', False, 0.0, None
+        elif key == "func_tol.criticality_measure":
+            type_str, nonetype_ok, lower, upper = 'float', False, 0.0, 1.0
+        elif key == "func_tol.tr_step":
+            type_str, nonetype_ok, lower, upper = 'float', False, 0.0, 1.0
+        elif key == "func_tol.max_iters":
+            type_str, nonetype_ok, lower, upper = 'int', False, 0, None
+        elif key == "sfista.max_iters_scaling":
+            type_str, nonetype_ok, lower, upper = 'float', False, 1.0, None
         else:
             assert False, "ParameterList.param_type() has unknown key: %s" % key
         return type_str, nonetype_ok, lower, upper