DFO-LS 1.2.1__py3-none-any.whl → 1.5.0__py3-none-any.whl

dfols/model.py

@@ -36,13 +36,16 @@ import numpy as np
 import scipy.linalg as LA
 
 from .trust_region import trsbox_geometry
-from .util import sumsq
+from .util import sumsq, dykstra, remove_scaling
 
 __all__ = ['Model']
 
+module_logger = logging.getLogger(__name__) 
+
+
 class Model(object):
-    def __init__(self, npt, x0, r0, xl, xu, r0_nsamples, n=None, m=None, abs_tol=1e-12, rel_tol=1e-20, precondition=True,
-                 do_logging=True):
+    def __init__(self, npt, x0, r0, xl, xu, projections, r0_nsamples, h=None, argsh=(), n=None, m=None, abs_tol=1e-12, rel_tol=1e-20, precondition=True,
+                 do_logging=True, scaling_changes=None):
         if n is None:
             n = len(x0)
         if m is None:
@@ -53,27 +56,35 @@ class Model(object):
         assert xu.shape == (n,), "xu has wrong shape (got %s, expect (%g,))" % (str(xu.shape), n)
         assert r0.shape == (m,), "r0 has wrong shape (got %s, expect (%g,))" % (str(r0.shape), m)
         self.do_logging = do_logging
+        self.scaling_changes = scaling_changes
         self.dim = n
         self.resid_dim = m
         self.num_pts = npt
         self.npt_so_far = 1  # number of points added so far (with function values)
 
+        self.h = h
+        self.argsh = argsh
+
         # Initialise to blank some useful stuff
         # Interpolation points
         self.xbase = x0.copy()
         self.sl = xl - self.xbase  # lower bound w.r.t. xbase (require xpt >= sl)
         self.su = xu - self.xbase  # upper bound w.r.t. xbase (require xpt <= su)
+        self.projections = projections
         self.points = np.zeros((npt, n))  # interpolation points w.r.t. xbase
 
         # Function values
         self.fval_v = np.inf * np.ones((npt, m))  # residuals for each xpt
         self.fval_v[0, :] = r0.copy()
-        self.fval = np.inf * np.ones((npt, ))  # overall objective value for each xpt
-        self.fval[0] = sumsq(r0)
+        
+        self.objval = np.inf * np.ones((npt, ))  # overall objective value for each xpt
+        self.objval[0] = sumsq(r0)
+        if h is not None:
+            self.objval[0] += h(remove_scaling(x0, self.scaling_changes), *argsh)
         self.kopt = 0  # index of current iterate (should be best value so far)
-        self.nsamples = np.zeros((npt,), dtype=np.int)  # number of samples used to evaluate objective at each point
+        self.nsamples = np.zeros((npt,), dtype=int)  # number of samples used to evaluate objective at each point
         self.nsamples[0] = r0_nsamples
-        self.fbeg = self.fval[0]  # f(x0), saved to check for sufficient reduction
+        self.objbeg = self.objval[0]  # f(x0), saved to check for sufficient reduction
 
         # Termination criteria
         self.abs_tol = abs_tol
@@ -86,7 +97,7 @@ class Model(object):
         # Saved point (in absolute coordinates) - always check this value before quitting solver
         self.xsave = None
         self.rsave = None
-        self.fsave = None
+        self.objsave = None
         self.jacsave = None
         self.nsamples_save = None
 
@@ -114,8 +125,8 @@ class Model(object):
     def ropt(self):
         return self.fval_v[self.kopt, :]  # residuals for current iterate
 
-    def fopt(self):
-        return self.fval[self.kopt]
+    def objopt(self):
+        return self.objval[self.kopt]
 
     def xpt(self, k, abs_coordinates=False):
         assert 0 <= k < self.npt(), "Invalid index %g" % k
@@ -123,18 +134,22 @@ class Model(object):
             return np.minimum(np.maximum(self.sl, self.points[k, :].copy()), self.su)
         else:
             # Apply bounds and convert back to absolute coordinates
+            if self.projections:
+                return dykstra(self.projections, self.xbase + self.points[k,:])
             return self.xbase + np.minimum(np.maximum(self.sl, self.points[k, :]), self.su)
 
     def rvec(self, k):
         assert 0 <= k < self.npt(), "Invalid index %g" % k
         return self.fval_v[k, :]
 
-    def fval(self, k):
+    def objval(self, k):
         assert 0 <= k < self.npt(), "Invalid index %g" % k
-        return self.fval[k]
+        return self.objval[k]
 
-    def as_absolute_coordinates(self, x):
+    def as_absolute_coordinates(self, x, full_dykstra=False):
         # If x were an interpolation point, get the absolute coordinates of x
+        if self.projections:
+            return dykstra(self.projections, self.xbase + x)
         return self.xbase + np.minimum(np.maximum(self.sl, x), self.su)
 
     def xpt_directions(self, include_kopt=True):
@@ -169,18 +184,20 @@ class Model(object):
 
         self.points[k, :] = x.copy()
         self.fval_v[k, :] = rvec.copy()
-        self.fval[k] = sumsq(rvec)
+        self.objval[k] = sumsq(rvec)
+        if self.h is not None:
+            self.objval[k] += self.h(remove_scaling(self.xbase + x, self.scaling_changes), *self.argsh)
         self.nsamples[k] = 1
         self.factorisation_current = False
 
-        if allow_kopt_update and self.fval[k] < self.fopt():
+        if allow_kopt_update and self.objval[k] < self.objopt():
             self.kopt = k
         return
 
     def swap_points(self, k1, k2):
         self.points[[k1, k2], :] = self.points[[k2, k1], :]
         self.fval_v[[k1, k2], :] = self.fval_v[[k2, k1], :]
-        self.fval[[k1, k2]] = self.fval[[k2, k1]]
+        self.objval[[k1, k2]] = self.objval[[k2, k1]]
         if self.kopt == k1:
             self.kopt = k2
         elif self.kopt == k2:
@@ -193,22 +210,27 @@ class Model(object):
         assert 0 <= k < self.npt(), "Invalid index %g" % k
         t = float(self.nsamples[k]) / float(self.nsamples[k] + 1)
         self.fval_v[k, :] = t * self.fval_v[k, :] + (1 - t) * rvec_extra
-        self.fval[k] = sumsq(self.fval_v[k, :])
+        # NOTE: how to sample when we have h? still at xpt(k), then add h(xpt(k)). Modify test if incorrect!
+        self.objval[k] = sumsq(self.fval_v[k, :])
+        if self.h is not None:
+            self.objval[k] += self.h(remove_scaling(self.xbase + self.points[k, :], self.scaling_changes), *self.argsh)
         self.nsamples[k] += 1
 
-        self.kopt = np.argmin(self.fval[:self.npt()])  # make sure kopt is always the best value we have
+        self.kopt = np.argmin(self.objval[:self.npt()])  # make sure kopt is always the best value we have
         return
 
     def add_new_point(self, x, rvec):
         self.points = np.append(self.points, x.reshape((1, self.n())), axis=0)  # append row to xpt
         self.fval_v = np.append(self.fval_v, rvec.reshape((1, self.m())), axis=0)  # append row to fval_v
-        f = np.dot(rvec, rvec)
-        self.fval = np.append(self.fval, f)  # append entry to fval
+        obj = sumsq(rvec)
+        if self.h is not None:
+            obj += self.h(remove_scaling(self.xbase + x, self.scaling_changes), *self.argsh)
+        self.objval = np.append(self.objval, obj)  # append entry to fval
         self.nsamples = np.append(self.nsamples, 1)  # add new sample number
         self.num_pts += 1  # make sure npt is updated
         self.npt_so_far += 1
 
-        if f < self.fopt():
+        if obj < self.objopt():
             self.kopt = self.npt() - 1
 
         self.factorisation_current = False
@@ -228,11 +250,14 @@ class Model(object):
         return
 
     def save_point(self, x, rvec, nsamples, x_in_abs_coords=True):
-        f = sumsq(rvec)
-        if self.fsave is None or f <= self.fsave:
-            self.xsave = x.copy() if x_in_abs_coords else self.as_absolute_coordinates(x)
+        xabs = x.copy() if x_in_abs_coords else self.as_absolute_coordinates(x)
+        obj = sumsq(rvec)
+        if self.h is not None:
+            obj += self.h(remove_scaling(xabs, self.scaling_changes), *self.argsh)
+        if self.objsave is None or obj <= self.objsave:
+            self.xsave = xabs
             self.rsave = rvec.copy()
-            self.fsave = f
+            self.objsave = obj
             self.jacsave = self.model_jac.copy()
             self.nsamples_save = nsamples
             return True
@@ -240,15 +265,15 @@ class Model(object):
             return False  # this value is worse than what we have already - didn't save
 
     def get_final_results(self):
-        # Return x and fval for optimal point (either from xsave+fsave or kopt)
-        if self.fsave is None or self.fopt() <= self.fsave:  # optimal has changed since xsave+fsave were last set
-            return self.xopt(abs_coordinates=True).copy(), self.ropt().copy(), self.fopt(), self.model_jac.copy(), self.nsamples[self.kopt]
+        # Return x and objval for optimal point (either from xsave+objsave or kopt)
+        if self.objsave is None or self.objopt() <= self.objsave:  # optimal has changed since xsave+objsave were last set
+            return self.xopt(abs_coordinates=True).copy(), self.ropt().copy(), self.objopt(), self.model_jac.copy(), self.nsamples[self.kopt]
         else:
-            return self.xsave.copy(), self.rsave.copy(), self.fsave, self.jacsave, self.nsamples_save
+            return self.xsave.copy(), self.rsave.copy(), self.objsave, self.jacsave, self.nsamples_save
 
     def min_objective_value(self):
         # Get termination criterion for f small: f <= abs_tol or f <= rel_tol * f0
-        return max(self.abs_tol, self.rel_tol * self.fbeg)
+        return max(self.abs_tol, self.rel_tol * self.objbeg)
 
     def model_value(self, d, d_based_at_xopt=True, with_const_term=False):
         if d_based_at_xopt:
@@ -301,12 +326,12 @@ class Model(object):
                 return col_scale(LA.solve_triangular(self.R, Qb), self.right_scaling)
         else:
             if self.do_logging:
-                logging.warning("model.solve_geom_system not using factorisation")
+                module_logger.warning("model.solve_geom_system not using factorisation")
             W, left_scaling, right_scaling = self.interpolation_matrix()
             return col_scale(LA.lstsq(W, col_scale(rhs * left_scaling))[0], right_scaling)
 
     def interpolate_mini_models_svd(self, verbose=False, make_full_rank=False, min_sing_val=1e-6, sing_val_frac=1.0, max_jac_cond=1e8,
-                                    get_chg_J=False):
+                                    get_chg_J=False, throw_error_on_nans=False):
         W, left_scaling, right_scaling = self.interpolation_matrix()
         self.factorise_geom_system()
         ls_interp_cond_num = np.linalg.cond(W) if verbose else 0.0  # scipy.linalg does not have condition number!
@@ -327,12 +352,18 @@ class Model(object):
             self.model_jac = np.dot(self.model_jac, np.dot(Qhat, Qhat.T))
 
         rhs = self.fval_v[fval_row_idx, :]  # size npt * m
+        if np.any(np.isnan(rhs)) and throw_error_on_nans:
+            if self.do_logging:
+                module_logger.warning("model.interpolate_mini_models_svd: NaNs encountered in objective evaluations, raising error")
+            raise np.linalg.LinAlgError("NaN encountered in objective evaluations")
         try:
             dg = self.solve_geom_system(rhs)  # size (n+1)*m
         except LA.LinAlgError:
             return False, None, None, None, None  # flag error
         except ValueError:
             return False, None, None, None, None  # flag error (e.g. inf or NaN encountered)
+        if not np.all(np.isfinite(dg)):  # another check for inf or NaN
+            return False, None, None, None, None
         J_old = self.model_jac.copy()
         self.model_jac = dg[1:,:].T
         self.model_const = dg[0,:] - np.dot(self.model_jac, xopt)  # shift base to xbase
@@ -361,7 +392,7 @@ class Model(object):
         return True, interp_error, sqrt(norm_J_error), linalg_resid, ls_interp_cond_num  # flag ok
 
     def build_full_model(self):
-        # Build full least squares objective model from mini-models
+        # Build full least squares model from mini-models
         # Centred around xopt
         r = self.model_const + np.dot(self.model_jac, self.xopt())  # constant term (for inexact interpolation)
         J = self.model_jac

dfols/params.py

@@ -44,6 +44,7 @@ class ParameterList(object):
         self.params["init.random_directions_make_orthogonal"] = True  # although random > orthogonal, avoid for init
         # Interpolation
         self.params["interpolation.precondition"] = True
+        self.params["interpolation.throw_error_on_nans"] = False  # throw numpy.linalg.LinAlgError if interpolating to nan data?
         # Logging
         self.params["logging.n_to_print_whole_x_vector"] = 6
         self.params["logging.save_diagnostic_info"] = False
@@ -81,7 +82,7 @@ class ParameterList(object):
         self.params["restarts.use_soft_restarts"] = True
         self.params["restarts.soft.num_geom_steps"] = 3
         self.params["restarts.soft.move_xk"] = True
-        self.params["restarts.soft.max_fake_successful_steps"] = maxfun  # number ratio>0 steps below fsave allowed
+        self.params["restarts.soft.max_fake_successful_steps"] = maxfun  # number ratio>0 steps below objsave allowed
         self.params["restarts.hard.use_old_rk"] = True  # recycle r(xk) from previous run?
         self.params["restarts.increase_npt"] = False
         self.params["restarts.increase_npt_amt"] = 1
@@ -108,7 +109,20 @@ class ParameterList(object):
         self.params["growing.full_rank.min_sing_val"] = 1e-6  # absolute floor on singular values
         self.params["growing.full_rank.svd_max_jac_cond"] = 1e8  # maximum condition number of Jacobian
         self.params["growing.perturb_trust_region_step"] = False  # add random direction onto TRS solution?
-
+        
+        # Dykstra's algorithm
+        self.params["dykstra.d_tol"] = 1e-10
+        self.params["dykstra.max_iters"] = 100
+        
+        # Matrix rank algorithm
+        self.params["matrix_rank.r_tol"] = 1e-18
+        
+        # Function tolerance when applying S-FISTA method
+        self.params["func_tol.criticality_measure"] = 1e-3
+        self.params["func_tol.tr_step"] = 1-1e-1
+        self.params["func_tol.max_iters"] = 500
+        self.params["sfista.max_iters_scaling"] = 2.0
+        
         self.params_changed = {}
         for p in self.params:
             self.params_changed[p] = False
@@ -142,6 +156,8 @@ class ParameterList(object):
             type_str, nonetype_ok, lower, upper = 'bool', False, None, None
         elif key == "interpolation.precondition":
             type_str, nonetype_ok, lower, upper = 'bool', False, None, None
+        elif key == "interpolation.throw_error_on_nans":
+            type_str, nonetype_ok, lower, upper = 'bool', False, None, None
         elif key == "logging.n_to_print_whole_x_vector":
             type_str, nonetype_ok, lower, upper = 'int', False, 0, None
         elif key == "logging.save_diagnostic_info":
@@ -254,6 +270,20 @@ class ParameterList(object):
             type_str, nonetype_ok, lower, upper = 'float', True, 1.0, None
         elif key == "growing.perturb_trust_region_step":
             type_str, nonetype_ok, lower, upper = 'bool', False, None, None
+        elif key == "dykstra.d_tol":
+            type_str, nonetype_ok, lower, upper = 'float', False, 0.0, None
+        elif key == "dykstra.max_iters":
+            type_str, nonetype_ok, lower, upper = 'int', False, 0, None
+        elif key == "matrix_rank.r_tol":
+            type_str, nonetype_ok, lower, upper = 'float', False, 0.0, None
+        elif key == "func_tol.criticality_measure":
+            type_str, nonetype_ok, lower, upper = 'float', False, 0.0, 1.0
+        elif key == "func_tol.tr_step":
+            type_str, nonetype_ok, lower, upper = 'float', False, 0.0, 1.0
+        elif key == "func_tol.max_iters":
+            type_str, nonetype_ok, lower, upper = 'int', False, 0, None
+        elif key == "sfista.max_iters_scaling":
+            type_str, nonetype_ok, lower, upper = 'float', False, 1.0, None
         else:
             assert False, "ParameterList.param_type() has unknown key: %s" % key
         return type_str, nonetype_ok, lower, upper