DFO-LS 1.4.1__py3-none-any.whl → 1.5.1__py3-none-any.whl

dfols/solver.py

@@ -48,10 +48,10 @@ module_logger = logging.getLogger(__name__)
 
 # A container for the results of the optimization routine
 class OptimResults(object):
-    def __init__(self, xmin, rmin, fmin, jacmin, nf, nx, nruns, exit_flag, exit_msg):
+    def __init__(self, xmin, rmin, objmin, jacmin, nf, nx, nruns, exit_flag, exit_msg, xmin_eval_num, jacmin_eval_nums):
         self.x = xmin
         self.resid = rmin
-        self.f = fmin
+        self.obj = objmin
         self.jacobian = jacmin
         self.nf = nf
         self.nx = nx
@@ -59,6 +59,8 @@ class OptimResults(object):
         self.flag = exit_flag
         self.msg = exit_msg
         self.diagnostic_info = None
+        self.xmin_eval_num = xmin_eval_num
+        self.jacmin_eval_nums = jacmin_eval_nums
         # Set standard names for exit flags
         self.EXIT_SLOW_WARNING = EXIT_SLOW_WARNING
         self.EXIT_MAXFUN_WARNING = EXIT_MAXFUN_WARNING
@@ -77,7 +79,7 @@ class OptimResults(object):
                 output += "Residual vector = %s\n" % str(self.resid)
             else:
                 output += "Not showing residual vector because it is too long; check self.resid\n"
-            output += "Objective value f(xmin) = %.10g\n" % self.f
+            output += "Objective value f(xmin) = %.10g\n" % self.obj
             output += "Needed %g objective evaluations (at %g points)\n" % (self.nf, self.nx)
             if self.nruns > 1:
                 output += "Did a total of %g runs\n" % self.nruns
@@ -89,14 +91,17 @@ class OptimResults(object):
                 output += "Not showing approximate Jacobian because it is too long; check self.jacobian\n"
             if self.diagnostic_info is not None:
                 output += "Diagnostic information available; check self.diagnostic_info\n"
+            output += "Solution xmin was evaluation point %g\n" % self.xmin_eval_num
+            if len(self.jacmin_eval_nums) < 100:
+                output += "Approximate Jacobian formed using evaluation points %s\n" % str(self.jacmin_eval_nums)
         output += "Exit flag = %g\n" % self.flag
         output += "%s\n" % self.msg
         output += "****************************\n"
         return output
 
 
-def solve_main(objfun, x0, args, xl, xu, projections, npt, rhobeg, rhoend, maxfun, nruns_so_far, nf_so_far, nx_so_far, nsamples, params,
-               diagnostic_info, scaling_changes, r0_avg_old=None, r0_nsamples_old=None, default_growing_method_set_by_user=None,
+def solve_main(objfun, x0, argsf, xl, xu, projections, npt, rhobeg, rhoend, maxfun, nruns_so_far, nf_so_far, nx_so_far, nsamples, params,
+               diagnostic_info, scaling_changes, h=None, lh=None, argsh=(), prox_uh=None, argsprox=None, r0_avg_old=None, r0_nsamples_old=None, default_growing_method_set_by_user=None,
                do_logging=True, print_progress=False):
     # Evaluate at x0 (keep nf, nx correct and check for f < 1e-12)
     # The hard bit is determining what m = len(r0) should be, and allocating memory appropriately
@@ -105,18 +110,17 @@ def solve_main(objfun, x0, args, xl, xu, projections, npt, rhobeg, rhoend, maxfu
         # Evaluate the first time...
         nf = nf_so_far + 1
         nx = nx_so_far + 1
-        r0, f0 = eval_least_squares_objective(objfun, remove_scaling(x0, scaling_changes),
-                                              args=args, eval_num=nf, pt_num=nx,
+        r0, obj0 = eval_least_squares_with_regularisation(objfun, remove_scaling(x0, scaling_changes), h, 
+                                              argsf=argsf, argsh=argsh, verbose=do_logging, eval_num=nf, pt_num=nx,
                                               full_x_thresh=params("logging.n_to_print_whole_x_vector"),
-                                              check_for_overflow=params("general.check_objfun_for_overflow"),
-                                              verbose=do_logging)
+                                              check_for_overflow=params("general.check_objfun_for_overflow"))
         m = len(r0)
 
         # Now we have m, we can evaluate the rest of the times
         rvec_list = np.zeros((number_of_samples, m))
-        f_list = np.zeros((number_of_samples,))
+        obj_list = np.zeros((number_of_samples,))
         rvec_list[0, :] = r0
-        f_list[0] = f0
+        obj_list[0] = obj0
         num_samples_run = 1
         exit_info = None
 
@@ -128,15 +132,20 @@ def solve_main(objfun, x0, args, xl, xu, projections, npt, rhobeg, rhoend, maxfu
 
             nf += 1
             # Don't increment nx for x0 - we did this earlier
-            rvec_list[i, :], f_list[i] = eval_least_squares_objective(objfun, remove_scaling(x0, scaling_changes), args=args, eval_num=nf, pt_num=nx,
+            rvec_list[i, :], obj_list[i] = eval_least_squares_with_regularisation(objfun, remove_scaling(x0, scaling_changes), h, 
+                                                argsf=argsf, argsh=argsh, verbose=do_logging, eval_num=nf, pt_num=nx,
                                                 full_x_thresh=params("logging.n_to_print_whole_x_vector"),
-                                                check_for_overflow=params("general.check_objfun_for_overflow"),
-                                                verbose=do_logging)
+                                                check_for_overflow=params("general.check_objfun_for_overflow"))
             num_samples_run += 1
 
         r0_avg = np.mean(rvec_list[:num_samples_run, :], axis=0)
-        if sumsq(r0_avg) <= params("model.abs_tol"):
-            exit_info = ExitInformation(EXIT_SUCCESS, "Objective is sufficiently small")
+        # NOTE: modify objvalue here
+        if h is None:
+            if sumsq(r0_avg) <= params("model.abs_tol"):
+                exit_info = ExitInformation(EXIT_SUCCESS, "Objective is sufficiently small")
+        else:
+            if sumsq(r0_avg) + h(remove_scaling(x0, scaling_changes), *argsh)<= params("model.abs_tol"):
+                exit_info = ExitInformation(EXIT_SUCCESS, "Objective is sufficiently small")
 
         if exit_info is not None:
             return x0, r0_avg, sumsq(r0_avg), None, num_samples_run, nf, nx, nruns_so_far+1, exit_info, diagnostic_info
@@ -162,8 +171,8 @@ def solve_main(objfun, x0, args, xl, xu, projections, npt, rhobeg, rhoend, maxfu
                 params('growing.delta_scale_new_dirns', new_value=0.1)
 
     # Initialise controller
-    control = Controller(objfun, args, x0, r0_avg, num_samples_run, xl, xu, projections, npt, rhobeg, rhoend, nf, nx, maxfun,
-                         params, scaling_changes, do_logging)
+    control = Controller(objfun, argsf, x0, r0_avg, num_samples_run, xl, xu, projections, npt, rhobeg, rhoend, nf, nx, maxfun,
+                         params, scaling_changes, do_logging, h=h, lh=lh, argsh=argsh,  prox_uh=prox_uh, argsprox=argsprox)
 
     # Initialise interpolation set
     number_of_samples = max(nsamples(control.delta, control.rho, 0, nruns_so_far), 1)
@@ -178,8 +187,8 @@ def solve_main(objfun, x0, args, xl, xu, projections, npt, rhobeg, rhoend, maxfu
             module_logger.info("Initialising (coordinate directions)")
         exit_info = control.initialise_coordinate_directions(number_of_samples, num_directions, params)
     if exit_info is not None:
-        x, rvec, f, jacmin, nsamples = control.model.get_final_results()
-        return x, rvec, f, None, nsamples, control.nf, control.nx, nruns_so_far + 1, exit_info, diagnostic_info
+        x, rvec, obj, jacmin, nsamples, x_eval_num, jac_eval_nums = control.model.get_final_results()
+        return x, rvec, obj, None, nsamples, control.nf, control.nx, nruns_so_far + 1, exit_info, diagnostic_info, x_eval_num, jac_eval_nums
 
     finished_growing = (control.model.npt() >= control.model.num_pts)  # have we finished growing the initial set yet?
 
@@ -271,16 +280,30 @@ def solve_main(objfun, x0, args, xl, xu, projections, npt, rhobeg, rhoend, maxfu
                 nruns_so_far += 1
                 break  # quit
 
-
-        # Trust region step
-        d, gopt, H, gnew, crvmin = control.trust_region_step(params)
+        tau = 1.0 # ratio used in the safety phase
+        if h is None:
+            # Trust region step
+            d, gopt, H, gnew, crvmin = control.trust_region_step(params)
+        else:
+            # Calculate criticality measure
+            criticality_measure = control.evaluate_criticality_measure(params)
+            # Trust region step
+            d, gopt, H, gnew, crvmin = control.trust_region_step(params, criticality_measure)
+            try:
+                tau = min(criticality_measure/(LA.norm(gopt)+lh), 1.0)
+            except ValueError:
+                # In some instances, gopt can have nan/inf values -- this ultimately calls a safety step and is generally fine
+                # but we need to set a value for tau nonetheless
+                tau = 1.0
+        
         if do_logging:
             module_logger.debug("Trust region step is d = " + str(d))
+        
         xnew = control.model.xopt() + d
         dnorm = min(LA.norm(d), control.delta)
 
         if print_progress:
-            print("{:^5}{:^7}{:^10.2e}{:^10.2e}{:^10.2e}{:^10.2e}{:^7}".format(nruns_so_far+1, current_iter+1, control.model.fopt(), np.linalg.norm(gopt), control.delta, control.rho, control.nf))
+            print("{:^5}{:^7}{:^10.2e}{:^10.2e}{:^10.2e}{:^10.2e}{:^7}".format(nruns_so_far+1, current_iter+1, control.model.objopt(), np.linalg.norm(gopt), control.delta, control.rho, control.nf))
 
         if params("logging.save_diagnostic_info"):
             diagnostic_info.save_info_from_control(control, nruns_so_far, current_iter,
@@ -289,7 +312,7 @@ def solve_main(objfun, x0, args, xl, xu, projections, npt, rhobeg, rhoend, maxfu
             diagnostic_info.update_interpolation_information(interp_error, ls_interp_cond_num, linalg_resid,
                                                              sqrt(norm_J_error), LA.norm(gopt), LA.norm(d))
 
-        if dnorm < params("general.safety_step_thresh") * control.rho and not finished_growing and params("growing.safety.do_safety_step"):
+        if dnorm < tau * params("general.safety_step_thresh") * control.rho and not finished_growing and params("growing.safety.do_safety_step"):
             if do_logging:
                 module_logger.debug("Safety step during growing phase")
 
@@ -415,10 +438,10 @@ def solve_main(objfun, x0, args, xl, xu, projections, npt, rhobeg, rhoend, maxfu
                 if do_logging:
                     module_logger.info("New rho = %g after %i function evaluations" % (control.rho, control.nf))
                     if control.n() < params("logging.n_to_print_whole_x_vector"):
-                        module_logger.debug("Best so far: f = %.15g at x = " % (control.model.fopt())
+                        module_logger.debug("Best so far: f = %.15g at x = " % (control.model.objopt())
                                       + str(control.model.xopt(abs_coordinates=True)))
                     else:
-                        module_logger.debug("Best so far: f = %.15g at x = [...]" % (control.model.fopt()))
+                        module_logger.debug("Best so far: f = %.15g at x = [...]" % (control.model.objopt()))
                 continue  # next iteration
             else:
                 # Quit on rho=rhoend
@@ -439,13 +462,14 @@ def solve_main(objfun, x0, args, xl, xu, projections, npt, rhobeg, rhoend, maxfu
                 else:
                     # Cannot reduce rho, so check xnew and quit
                     x = control.model.as_absolute_coordinates(xnew)
+                    ##print("x from xnew", x)
                     number_of_samples = max(nsamples(control.delta, control.rho, current_iter, nruns_so_far), 1)
-                    rvec_list, f_list, num_samples_run, exit_info = control.evaluate_objective(x, number_of_samples,
+                    rvec_list, obj_list, num_samples_run, exit_info = control.evaluate_objective(x, number_of_samples,
                                                                                                params)
                     
                     if num_samples_run > 0:
                             control.model.save_point(x, np.mean(rvec_list[:num_samples_run, :], axis=0),
-                                                     num_samples_run, x_in_abs_coords=True)
+                                                     num_samples_run, control.nx, x_in_abs_coords=True)
                     
                     if exit_info is not None:
                         nruns_so_far += 1
@@ -514,8 +538,9 @@ def solve_main(objfun, x0, args, xl, xu, projections, npt, rhobeg, rhoend, maxfu
 
             # Evaluate new point
             x = control.model.as_absolute_coordinates(xnew)
+            ##print("x from xnew again", x)
             number_of_samples = max(nsamples(control.delta, control.rho, current_iter, nruns_so_far), 1)
-            rvec_list, f_list, num_samples_run, exit_info = control.evaluate_objective(x, number_of_samples, params)
+            rvec_list, obj_list, num_samples_run, exit_info = control.evaluate_objective(x, number_of_samples, params)
             if np.any(np.isnan(rvec_list)):
                 # Just exit without saving the current point
                 # We should be able to do a hard restart though, because it's unlikely 
@@ -529,13 +554,13 @@ def solve_main(objfun, x0, args, xl, xu, projections, npt, rhobeg, rhoend, maxfu
                 break  # quit
             if exit_info is not None:
                 if num_samples_run > 0:
-                    control.model.save_point(x, np.mean(rvec_list[:num_samples_run, :], axis=0), num_samples_run,
+                    control.model.save_point(x, np.mean(rvec_list[:num_samples_run, :], axis=0), num_samples_run, control.nx,
                                              x_in_abs_coords=True)
                 nruns_so_far += 1
                 break  # quit
 
             # Estimate f in order to compute 'actual reduction'
-            ratio, exit_info = control.calculate_ratio(current_iter, rvec_list[:num_samples_run, :], d, gopt, H)
+            ratio, exit_info = control.calculate_ratio(control.model.xopt(abs_coordinates=True), current_iter, rvec_list[:num_samples_run, :], d, gopt, H)
             if exit_info is not None:
                 if exit_info.able_to_do_restart() and params("restarts.use_restarts") and params(
                         "restarts.use_soft_restarts"):
@@ -565,9 +590,9 @@ def solve_main(objfun, x0, args, xl, xu, projections, npt, rhobeg, rhoend, maxfu
                 diagnostic_info.update_slow_iter(-1)  # n/a, unless otherwise update
             if ratio < params("tr_radius.eta1"):  # ratio < 0.1
                 if finished_growing:
-                    control.delta = min(params("tr_radius.gamma_dec") * control.delta, dnorm)
+                    control.delta = min(params("tr_radius.gamma_dec") * control.delta, dnorm) / tau
                 else:
-                    control.delta = min(params("growing.gamma_dec") * control.delta, dnorm)  # different gamma_dec
+                    control.delta = min(params("growing.gamma_dec") * control.delta, dnorm) / tau  # different gamma_dec
                 if params("logging.save_diagnostic_info"):
                     diagnostic_info.update_iter_type(ITER_ACCEPTABLE_NO_GEOM if ratio > 0.0
                                                      else ITER_UNSUCCESSFUL_NO_GEOM)  # we flag geom update below
@@ -618,7 +643,7 @@ def solve_main(objfun, x0, args, xl, xu, projections, npt, rhobeg, rhoend, maxfu
 
             if do_logging:
                 module_logger.debug("Updating with knew = %i" % knew)
-            control.model.change_point(knew, xnew, rvec_list[0, :])  # expect step, not absolute x
+            control.model.change_point(knew, xnew, rvec_list[0, :], control.nx)  # expect step, not absolute x
             for i in range(1, num_samples_run):
                 control.model.add_new_sample(knew, rvec_extra=rvec_list[i, :])
 
@@ -651,7 +676,7 @@ def solve_main(objfun, x0, args, xl, xu, projections, npt, rhobeg, rhoend, maxfu
                         break  # quit
 
                 # Update list of successful steps
-                this_step_was_not_improvement = control.model.fsave is not None and control.model.fopt() > control.model.fsave
+                this_step_was_not_improvement = control.model.objsave is not None and control.model.objopt() > control.model.objsave
                 succ_steps_not_improvement.pop()  # remove last item
                 succ_steps_not_improvement.insert(0, this_step_was_not_improvement)  # add at beginning
                 # Terminate (not restart) if all are True
@@ -828,10 +853,10 @@ def solve_main(objfun, x0, args, xl, xu, projections, npt, rhobeg, rhoend, maxfu
                 if do_logging:
                     module_logger.info("New rho = %g after %i function evaluations" % (control.rho, control.nf))
                     if control.n() < params("logging.n_to_print_whole_x_vector"):
-                        module_logger.debug("Best so far: f = %.15g at x = " % (control.model.fopt())
+                        module_logger.debug("Best so far: f = %.15g at x = " % (control.model.objopt())
                                       + str(control.model.xopt(abs_coordinates=True)))
                     else:
-                        module_logger.debug("Best so far: f = %.15g at x = [...]" % (control.model.fopt()))
+                        module_logger.debug("Best so far: f = %.15g at x = [...]" % (control.model.objopt()))
                 continue  # next iteration
             else:
                 # Quit on rho=rhoend
@@ -857,14 +882,14 @@ def solve_main(objfun, x0, args, xl, xu, projections, npt, rhobeg, rhoend, maxfu
     # (end main loop)
 
     # Quit & return the important information
-    x, rvec, f, jacmin, nsamples = control.model.get_final_results()
+    x, rvec, obj, jacmin, nsamples, x_eval_num, jac_eval_nums = control.model.get_final_results()
     if do_logging:
         module_logger.debug("At return from DFO-LS, number of function evals = %i" % nf)
-        module_logger.debug("Smallest objective value = %.15g at x = " % f + str(x))
-    return x, rvec, f, jacmin, nsamples, control.nf, control.nx, nruns_so_far, exit_info, diagnostic_info
+        module_logger.debug("Smallest objective value = %.15g at x = " % obj + str(x))
+    return x, rvec, obj, jacmin, nsamples, control.nf, control.nx, nruns_so_far, exit_info, diagnostic_info, x_eval_num, jac_eval_nums
 
 
-def solve(objfun, x0, args=(), bounds=None, projections=[], npt=None, rhobeg=None, rhoend=1e-8, maxfun=None, nsamples=None, user_params=None,
+def solve(objfun, x0, h=None, lh=None, prox_uh=None, argsf=(), argsh=(), argsprox=(), bounds=None, projections=[], npt=None, rhobeg=None, rhoend=1e-8, maxfun=None, nsamples=None, user_params=None,
           objfun_has_noise=False, scaling_within_bounds=False, do_logging=True, print_progress=False):
     x0 = x0.astype(float)
     n = len(x0)
@@ -934,13 +959,21 @@ def solve(objfun, x0, args=(), bounds=None, projections=[], npt=None, rhobeg=Non
 
     exit_info = None
     # Input & parameter checks
+    if exit_info is None and h is not None:
+        if prox_uh is None:
+            exit_info = ExitInformation(EXIT_INPUT_ERROR, "Must provide prox_uh input if h is not None")
+        elif lh is None:
+            exit_info = ExitInformation(EXIT_INPUT_ERROR, "Must provide lh input if h is not None")
+        elif lh <= 0.0:
+            exit_info = ExitInformation(EXIT_INPUT_ERROR, "lh must be strictly positive")
+
     if exit_info is None and npt < n + 1:
         exit_info = ExitInformation(EXIT_INPUT_ERROR, "npt must be >= n+1 for linear models with inexact interpolation")
 
-    if exit_info is None and rhobeg < 0.0:
+    if exit_info is None and rhobeg <= 0.0:
         exit_info = ExitInformation(EXIT_INPUT_ERROR, "rhobeg must be strictly positive")
 
-    if exit_info is None and rhoend < 0.0:
+    if exit_info is None and rhoend <= 0.0:
         exit_info = ExitInformation(EXIT_INPUT_ERROR, "rhoend must be strictly positive")
 
     if exit_info is None and rhobeg <= rhoend:
@@ -1013,12 +1046,12 @@ def solve(objfun, x0, args=(), bounds=None, projections=[], npt=None, rhobeg=Non
             x0 = xp.copy()
 
     # Enforce lower & upper bounds on x0
-    idx = (x0 <= xl)
+    idx = (x0 < xl)
     if np.any(idx):
         warnings.warn("x0 below lower bound, adjusting", RuntimeWarning)
     x0[idx] = xl[idx]
 
-    idx = (x0 >= xu)
+    idx = (x0 > xu)
     if np.any(idx):
         warnings.warn("x0 above upper bound, adjusting", RuntimeWarning)
     x0[idx] = xu[idx]
@@ -1028,9 +1061,9 @@ def solve(objfun, x0, args=(), bounds=None, projections=[], npt=None, rhobeg=Non
     nruns = 0
     nf = 0
     nx = 0
-    xmin, rmin, fmin, jacmin, nsamples_min, nf, nx, nruns, exit_info, diagnostic_info = \
-        solve_main(objfun, x0, args, xl, xu, projections, npt, rhobeg, rhoend, maxfun, nruns, nf, nx, nsamples, params,
-                    diagnostic_info, scaling_changes, default_growing_method_set_by_user=default_growing_method_set_by_user,
+    xmin, rmin, objmin, jacmin, nsamples_min, nf, nx, nruns, exit_info, diagnostic_info, xmin_eval_num, jacmin_eval_nums = \
+        solve_main(objfun, x0, argsf, xl, xu, projections, npt, rhobeg, rhoend, maxfun, nruns, nf, nx, nsamples, params,
+                    diagnostic_info, scaling_changes, h, lh, argsh, prox_uh, argsprox, default_growing_method_set_by_user=default_growing_method_set_by_user,
                    do_logging=do_logging, print_progress=print_progress)
 
     # Hard restarts loop
@@ -1045,27 +1078,28 @@ def solve(objfun, x0, args=(), bounds=None, projections=[], npt=None, rhobeg=Non
 
         if do_logging:
             module_logger.info("Restarting from finish point (f = %g) after %g function evals; using rhobeg = %g and rhoend = %g"
-                     % (fmin, nf, rhobeg, rhoend))
+                     % (objmin, nf, rhobeg, rhoend))
         if params("restarts.hard.use_old_rk"):
-            xmin2, rmin2, fmin2, jacmin2, nsamples2, nf, nx, nruns, exit_info, diagnostic_info = \
-                solve_main(objfun, xmin, args, xl, xu, projections, npt, rhobeg, rhoend, maxfun, nruns, nf, nx, nsamples, params,
-                            diagnostic_info, scaling_changes, r0_avg_old=rmin, r0_nsamples_old=nsamples_min,
+            xmin2, rmin2, objmin2, jacmin2, nsamples2, nf, nx, nruns, exit_info, diagnostic_info, xmin_eval_num2, jacmin_eval_nums2 = \
+                solve_main(objfun, xmin, argsf, xl, xu, projections, npt, rhobeg, rhoend, maxfun, nruns, nf, nx, nsamples, params,
+                            diagnostic_info, scaling_changes, h, lh, argsh, prox_uh, argsprox, r0_avg_old=rmin, r0_nsamples_old=nsamples_min,
                            do_logging=do_logging, print_progress=print_progress)
         else:
-            xmin2, rmin2, fmin2, jacmin2, nsamples2, nf, nx, nruns, exit_info, diagnostic_info = \
-                solve_main(objfun, xmin, args, xl, xu, projections, npt, rhobeg, rhoend, maxfun, nruns, nf, nx, nsamples, params,
-                           diagnostic_info, scaling_changes, do_logging=do_logging, print_progress=print_progress)
+            xmin2, rmin2, objmin2, jacmin2, nsamples2, nf, nx, nruns, exit_info, diagnostic_info, xmin_eval_num2, jacmin_eval_nums2 = \
+                solve_main(objfun, xmin, argsf, xl, xu, projections, npt, rhobeg, rhoend, maxfun, nruns, nf, nx, nsamples, params,
+                           diagnostic_info, scaling_changes, h, lh, argsh, prox_uh, argsprox, do_logging=do_logging, print_progress=print_progress)
 
-        if fmin2 < fmin or np.isnan(fmin):
+        if objmin2 < objmin or np.isnan(objmin):
             if do_logging:
-                module_logger.info("Successful run with new f = %s compared to old f = %s" % (fmin2, fmin))
+                module_logger.info("Successful run with new f = %s compared to old f = %s" % (objmin2, objmin))
             last_successful_run = nruns
-            (xmin, rmin, fmin, nsamples_min) = (xmin2, rmin2, fmin2, nsamples2)
+            (xmin, rmin, objmin, nsamples_min, xmin_eval_num) = (xmin2, rmin2, objmin2, nsamples2, xmin_eval_num2)
             if jacmin2 is not None:  # may be None if finished during setup phase, in which case just use old Jacobian
                 jacmin = jacmin2
+                jacmin_eval_nums = jacmin_eval_nums2
         else:
             if do_logging:
-                module_logger.info("Unsuccessful run with new f = %s compared to old f = %s" % (fmin2, fmin))
+                module_logger.info("Unsuccessful run with new f = %s compared to old f = %s" % (objmin2, objmin))
 
     if nruns - last_successful_run >= params("restarts.max_unsuccessful_restarts"):
         exit_info = ExitInformation(EXIT_SUCCESS, "Reached maximum number of unsuccessful restarts")
@@ -1077,7 +1111,7 @@ def solve(objfun, x0, args=(), bounds=None, projections=[], npt=None, rhobeg=Non
     if scaling_changes is not None and jacmin is not None:
         for i in range(n):
             jacmin[:, i] = jacmin[:, i] / scaling_changes[1][i]
-    results = OptimResults(remove_scaling(xmin, scaling_changes), rmin, fmin, jacmin, nf, nx, nruns, exit_flag, exit_msg)
+    results = OptimResults(remove_scaling(xmin, scaling_changes), rmin, objmin, jacmin, nf, nx, nruns, exit_flag, exit_msg, xmin_eval_num, jacmin_eval_nums)
     if params("logging.save_diagnostic_info"):
         df = diagnostic_info.to_dataframe(with_xk=params("logging.save_xk"), with_rk=params("logging.save_rk"))
         results.diagnostic_info = df

dfols/trust_region.py

@@ -29,14 +29,14 @@ solves
     s.t.  lower <= x <= upper
           ||x-xbase|| <= Delta
 With this value, the variable d=x-xbase solves the problem
-    min_s  abs(c + g' * d)
+    min_d  abs(c + g' * d)
     s.t.   lower <= xbase + d <= upper
           ||d|| <= delta
 Again, we have a version of this for handling arbitrary constraints
 The call
     x = ctrsbox_geometry(xbase, c, g, projections, Delta)
 Solves
-    min_s  abs(c + g' * d)
+    min_d  abs(c + g' * d)
     s.t.   xbase + d is feasible w.r.t. the constraint set C
           ||d|| <= delta
 
@@ -70,7 +70,7 @@ alternative licensing.
 # Ensure compatibility with Python 2
 from __future__ import absolute_import, division, print_function, unicode_literals
 
-from math import sqrt
+from math import sqrt, ceil
 import numpy as np
 try:
     import trustregion
@@ -79,13 +79,93 @@ except ImportError:
     # Fall back to Python implementation
     USE_FORTRAN = False
 
-from .util import dykstra, pball, pbox, sumsq, model_value
+from .util import dykstra, pball, pbox, sumsq, model_value, remove_scaling
 
-__all__ = ['ctrsbox', 'ctrsbox_geometry', 'trsbox', 'trsbox_geometry']
+__all__ = ['ctrsbox_sfista', 'ctrsbox_pgd', 'ctrsbox_geometry', 'trsbox', 'trsbox_geometry']
 
 ZERO_THRESH = 1e-14
 
-def ctrsbox(xopt, g, H, projections, delta, d_max_iters=100, d_tol=1e-10, use_fortran=USE_FORTRAN):
+def ctrsbox_sfista(xopt, g, H, projections, delta, h, L_h, prox_uh, argsh=(), argsprox=(), func_tol=1e-3, max_iters=500, d_max_iters=100, d_tol=1e-10, use_fortran=USE_FORTRAN, scaling_changes=None, sfista_iters_scale=1.0):
+    n = xopt.size
+    assert xopt.shape == (n,), "xopt has wrong shape (should be vector)"
+    assert g.shape == (n,), "g and xopt have incompatible sizes"
+    assert len(H.shape) == 2, "H must be a matrix"
+    assert H.shape == (n,n), "H and xopt have incompatible sizes"
+    assert np.allclose(H, H.T), "H must be symmetric"
+    assert delta > 0.0, "delta must be strictly positive"
+
+    # Initialization
+    d = np.zeros(n) # start with zero vector
+    y = np.zeros(n)
+    t = 1
+    k_H = np.linalg.norm(H, 2)
+    crvmin = -1.0
+    
+    # Number of iterations & smoothing parameter, from Theorem 10.57 in 
+    #   [A. Beck. First-order methods in optimization, SIAM, 2017]
+    # We do not use the values of k and mu given in the theorem statement, but rather the intermediate
+    # results on p313 (K1 for number of iterations, and the immediate next line for mu)
+    # Note: in the book's notation, Gamma=delta^2, alpha=1, beta=L_h^2/2, Lf=k_H [alpha and beta from Thm 10.51]
+    try:
+        MAX_LOOP_ITERS = ceil(sfista_iters_scale * delta * (L_h+sqrt(L_h*L_h+2*k_H*func_tol)) / func_tol)
+        MAX_LOOP_ITERS = min(MAX_LOOP_ITERS, max_iters)
+    except ValueError:
+        MAX_LOOP_ITERS = max_iters
+    u =  2 * delta / (MAX_LOOP_ITERS * L_h) # smoothing parameter
+    # u = 2 * func_tol / (L_h ** 2 + L_h * sqrt(L_h ** 2 + 2 * k_H * func_tol))  # the above choice works better in practice
+
+    def gradient_Fu(xopt, g, H, u, prox_uh, d):
+    # Calculate gradient_Fu,
+    # where Fu(d) := g(d) + h_u(d) and h_u(d) is a 1/u-smooth approximation of h.
+    # We assume that h is globally Lipschitz continous with constant L_h,
+    # then we can let h_u(d) be the Moreau Envelope M_h_u(d) of h.
+        return g + H @ d + (xopt + d - prox_uh(remove_scaling(xopt + d, scaling_changes), u, *argsprox)) / u
+
+    # Lipschitz constant of gradient_Fu
+    l = k_H + 1 / u 
+
+    # trust region is a ball of radius delta around xopt
+    trproj = lambda w: pball(w, xopt, delta)
+
+    # combine trust region constraints with user-entered constraints
+    P = list(projections)  # make a copy of the projections list
+    P.append(trproj)
+    def proj(d0):
+        p = dykstra(P, xopt+d0, max_iter=d_max_iters, tol=d_tol)
+        # we want the step only, so we subtract xopt
+        # from the new point: proj(xk+d) - xk
+        return p - xopt
+
+    # general step
+    model_value_best = model_value(g, H, d, xopt, h, argsh, scaling_changes)
+    d_best = d.copy()
+    for k in range(MAX_LOOP_ITERS):
+        prev_d = d.copy()
+        prev_t = t
+        # gradient_Fu at y
+        g_Fu = gradient_Fu(xopt, g, H, u, prox_uh, d, *argsprox)
+
+        # main update step
+        d = proj(y - g_Fu / l)
+        new_model_value = model_value(g, H, d, xopt, h, argsh, scaling_changes)
+        if new_model_value < model_value_best:
+            d_best = d.copy()
+            model_value_best = new_model_value
+
+        # update true gradient
+        # gnew is the gradient of the smoothed function
+        gnew = gradient_Fu(xopt, g, H, u, prox_uh, d, *argsprox)
+
+        # update CRVMIN
+        crv = d.dot(H).dot(d)/sumsq(d) if sumsq(d) >= ZERO_THRESH else crvmin
+        crvmin = min(crvmin, crv) if crvmin != -1.0 else crv
+        
+        # momentum update
+        t = (1 + sqrt(1 + 4*t*t)) / 2
+        y = d + (prev_t - 1) * (d - prev_d) / t
+    return d, gnew, crvmin
+
+def ctrsbox_pgd(xopt, g, H, projections, delta, d_max_iters=100, d_tol=1e-10, use_fortran=USE_FORTRAN):
     n = xopt.size
     assert xopt.shape == (n,), "xopt has wrong shape (should be vector)"
     assert g.shape == (n,), "g and xopt have incompatible sizes"
@@ -151,7 +231,6 @@ def ctrsbox(xopt, g, H, projections, delta, d_max_iters=100, d_tol=1e-10, use_fo
 
     return d, gnew, crvmin
 
-
 def trsbox(xopt, g, H, sl, su, delta, use_fortran=USE_FORTRAN):
     if use_fortran:
         return trustregion.solve(g, H, delta,

dfols/util.py

@@ -31,7 +31,7 @@ import scipy.linalg as LA
 import sys
 
 
-__all__ = ['sumsq', 'eval_least_squares_objective', 'model_value', 'random_orthog_directions_within_bounds',
+__all__ = ['sumsq', 'eval_least_squares_with_regularisation', 'model_value', 'random_orthog_directions_within_bounds',
            'random_directions_within_bounds', 'apply_scaling', 'remove_scaling', 'pbox', 'pball', 'dykstra', 'qr_rank']
 
 module_logger = logging.getLogger(__name__) 
@@ -47,9 +47,9 @@ def sumsq(x):
     return np.dot(x, x)
 
 
-def eval_least_squares_objective(objfun, x, args=(), verbose=True, eval_num=0, pt_num=0, full_x_thresh=6, check_for_overflow=True):
+def eval_least_squares_with_regularisation(objfun, x, h=None, argsf=(), argsh=(), verbose=True, eval_num=0, pt_num=0, full_x_thresh=6, check_for_overflow=True):
     # Evaluate least squares function
-    fvec = objfun(x, *args)
+    fvec = objfun(x, *argsf)
 
     if check_for_overflow:
         try:
@@ -62,20 +62,31 @@ def eval_least_squares_objective(objfun, x, args=(), verbose=True, eval_num=0, p
     else:
         f = sumsq(fvec)
 
+    # objective = least-squares + regularisation
+    obj = f
+    if h is not None:
+        # Evaluate regularisation term
+        hvalue = h(x, *argsh)
+        obj = f + hvalue
+
     if verbose:
         if len(x) < full_x_thresh:
-            module_logger.info("Function eval %i at point %i has f = %.15g at x = " % (eval_num, pt_num, f) + str(x))
+            module_logger.info("Function eval %i at point %i has obj = %.15g at x = " % (eval_num, pt_num, obj) + str(x))
         else:
-            module_logger.info("Function eval %i at point %i has f = %.15g at x = [...]" % (eval_num, pt_num, f))
+            module_logger.info("Function eval %i at point %i has obj = %.15g at x = [...]" % (eval_num, pt_num, obj))
 
-    return fvec, f
+    return fvec, obj
 
 
-def model_value(g, H, s):
-    # Calculate model value (s^T * g + 0.5* s^T * H * s) = s^T * (gopt + 0.5 * H*s)
+def model_value(g, H, s, xopt=(), h=None,argsh=(), scaling_changes=None):
+    # Calculate model value (s^T * g + 0.5* s^T * H * s) + h(xopt + s) = s^T * (gopt + 0.5 * H*s) + h(xopt + s)
     assert g.shape == s.shape, "g and s have incompatible sizes"
     Hs = H.dot(s)
-    return np.dot(s, g + 0.5*Hs)
+    rtn = np.dot(s, g + 0.5*Hs)
+    if h is not None:
+        hvalue = h(remove_scaling(xopt+s, scaling_changes), *argsh)
+        rtn += hvalue
+    return rtn
 
 
 def get_scale(dirn, delta, lower, upper):

DFO_LS-1.4.1.dist-info/RECORD

@@ -1,14 +0,0 @@
-dfols/__init__.py,sha256=D-x5glfZFfJ8-bdjA-4k4JFTDu1Eylaz3EL4GSH28eI,1605
-dfols/controller.py,sha256=LSeHZoKaKUEYgB1_2subjKskHJ8mWccMbn-LOpxJ7LM,42769
-dfols/diagnostic_info.py,sha256=2kEUkL-MS4eDENUf1r2hOWsntP8OxMDKi_kyHmrC9V4,6081
-dfols/hessian.py,sha256=sExx4J4KoGwHItbthX2odosB2ONbQFvLdlcod7PIh4k,4262
-dfols/model.py,sha256=q70zuqocNtsaXzNjWHcTdrS209BdQt4uY0GNtp0qlI8,18809
-dfols/params.py,sha256=_Va1ybnQDIzWaXvImcSeH8xnNE_A2zpAfBgDG74sc5c,17557
-dfols/solver.py,sha256=IKg3xWPLYlOW_zuTc_-HY_3ZvdDEfkyxARerERUQHlU,61264
-dfols/trust_region.py,sha256=hRKQx0fpSxol7dLZO0yrT7O5IDptPPSnDvxKQNZ3r0M,24603
-dfols/util.py,sha256=ysdIHTkrkWwCRKuGffofehKl-t5dT3sD9dfy0muI4ZI,9852
-DFO_LS-1.4.1.dist-info/LICENSE.txt,sha256=jOtLnuWt7d5Hsx6XXB2QxzrSe2sWWh3NgMfFRetluQM,35147
-DFO_LS-1.4.1.dist-info/METADATA,sha256=RR6KhJi4Ae_1PES8Bpzqm3AYK2w12V-2MyDyjaCDe80,8552
-DFO_LS-1.4.1.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
-DFO_LS-1.4.1.dist-info/top_level.txt,sha256=UfxRhaDN8HQx2_l17KbrDrERJ90OCN7VKkDMpYYbRLU,6
-DFO_LS-1.4.1.dist-info/RECORD,,