DFO-LS 1.2.1__py3-none-any.whl → 1.4.1__py3-none-any.whl

dfols/params.py

@@ -44,6 +44,7 @@ class ParameterList(object):
         self.params["init.random_directions_make_orthogonal"] = True  # although random > orthogonal, avoid for init
         # Interpolation
         self.params["interpolation.precondition"] = True
+        self.params["interpolation.throw_error_on_nans"] = False  # throw numpy.linalg.LinAlgError if interpolating to nan data?
         # Logging
         self.params["logging.n_to_print_whole_x_vector"] = 6
         self.params["logging.save_diagnostic_info"] = False
@@ -108,6 +109,11 @@ class ParameterList(object):
         self.params["growing.full_rank.min_sing_val"] = 1e-6  # absolute floor on singular values
         self.params["growing.full_rank.svd_max_jac_cond"] = 1e8  # maximum condition number of Jacobian
         self.params["growing.perturb_trust_region_step"] = False  # add random direction onto TRS solution?
+        # Dykstra's algorithm
+        self.params["dykstra.d_tol"] = 1e-10
+        self.params["dykstra.max_iters"] = 100
+        # Matrix rank algorithm
+        self.params["matrix_rank.r_tol"] = 1e-18
 
         self.params_changed = {}
         for p in self.params:
@@ -142,6 +148,8 @@ class ParameterList(object):
             type_str, nonetype_ok, lower, upper = 'bool', False, None, None
         elif key == "interpolation.precondition":
             type_str, nonetype_ok, lower, upper = 'bool', False, None, None
+        elif key == "interpolation.throw_error_on_nans":
+            type_str, nonetype_ok, lower, upper = 'bool', False, None, None
         elif key == "logging.n_to_print_whole_x_vector":
             type_str, nonetype_ok, lower, upper = 'int', False, 0, None
         elif key == "logging.save_diagnostic_info":
@@ -254,6 +262,12 @@ class ParameterList(object):
             type_str, nonetype_ok, lower, upper = 'float', True, 1.0, None
         elif key == "growing.perturb_trust_region_step":
             type_str, nonetype_ok, lower, upper = 'bool', False, None, None
+        elif key == "dykstra.d_tol":
+            type_str, nonetype_ok, lower, upper = 'float', False, 0.0, None
+        elif key == "dykstra.max_iters":
+            type_str, nonetype_ok, lower, upper = 'int', False, 0, None
+        elif key == "matrix_rank.r_tol":
+            type_str, nonetype_ok, lower, upper = 'float', False, 0.0, None
         else:
             assert False, "ParameterList.param_type() has unknown key: %s" % key
         return type_str, nonetype_ok, lower, upper

dfols/solver.py

@@ -43,6 +43,8 @@ from .util import *
 
 __all__ = ['solve']
 
+module_logger = logging.getLogger(__name__) 
+
 
 # A container for the results of the optimization routine
 class OptimResults(object):
@@ -93,7 +95,7 @@ class OptimResults(object):
         return output
 
 
-def solve_main(objfun, x0, args, xl, xu, npt, rhobeg, rhoend, maxfun, nruns_so_far, nf_so_far, nx_so_far, nsamples, params,
+def solve_main(objfun, x0, args, xl, xu, projections, npt, rhobeg, rhoend, maxfun, nruns_so_far, nf_so_far, nx_so_far, nsamples, params,
                diagnostic_info, scaling_changes, r0_avg_old=None, r0_nsamples_old=None, default_growing_method_set_by_user=None,
                do_logging=True, print_progress=False):
     # Evaluate at x0 (keep nf, nx correct and check for f < 1e-12)
@@ -153,14 +155,14 @@ def solve_main(objfun, x0, args, xl, xu, npt, rhobeg, rhoend, maxfun, nruns_so_f
         # However, this can fail for m<n, so need to use an alternative method (perturb_trust_region_step)
         if m < len(x0):
             if do_logging:
-                logging.debug("Inverse problem (m<n), switching default growing method")
+                module_logger.debug("Inverse problem (m<n), switching default growing method")
             params('growing.full_rank.use_full_rank_interp', new_value=False)
             params('growing.perturb_trust_region_step', new_value=True)
             if not params.params_changed['growing.delta_scale_new_dirns']:
                 params('growing.delta_scale_new_dirns', new_value=0.1)
 
     # Initialise controller
-    control = Controller(objfun, args, x0, r0_avg, num_samples_run, xl, xu, npt, rhobeg, rhoend, nf, nx, maxfun,
+    control = Controller(objfun, args, x0, r0_avg, num_samples_run, xl, xu, projections, npt, rhobeg, rhoend, nf, nx, maxfun,
                          params, scaling_changes, do_logging)
 
     # Initialise interpolation set
@@ -169,11 +171,11 @@ def solve_main(objfun, x0, args, xl, xu, npt, rhobeg, rhoend, maxfun, nruns_so_f
                          npt - 1)  # cap at npt
     if params("init.random_initial_directions"):
         if do_logging:
-            logging.info("Initialising (random directions)")
+            module_logger.info("Initialising (random directions)")
         exit_info = control.initialise_random_directions(number_of_samples, num_directions, params)
     else:
         if do_logging:
-            logging.info("Initialising (coordinate directions)")
+            module_logger.info("Initialising (coordinate directions)")
         exit_info = control.initialise_coordinate_directions(number_of_samples, num_directions, params)
     if exit_info is not None:
         x, rvec, f, jacmin, nsamples = control.model.get_final_results()
@@ -195,18 +197,18 @@ def solve_main(objfun, x0, args, xl, xu, npt, rhobeg, rhoend, maxfun, nruns_so_f
     # ------------------------------------------
     current_iter = -1
     if do_logging:
-        logging.info("Beginning main loop")
+        module_logger.info("Beginning main loop")
     if print_progress:
         print("{:^5}{:^7}{:^10}{:^10}{:^10}{:^10}{:^7}".format("Run", "Iter", "Obj", "Grad", "Delta", "rho", "Evals"))
     while True:
         current_iter += 1
 
         if do_logging:
-            logging.debug("*** Iter %g (delta = %g, rho = %g) ***" % (current_iter, control.delta, control.rho))
+            module_logger.debug("*** Iter %g (delta = %g, rho = %g) ***" % (current_iter, control.delta, control.rho))
 
         if (not finished_growing) and control.model.npt() >= control.model.num_pts:
             if do_logging:
-                logging.info("Finished growing init set")
+                module_logger.info("Finished growing init set")
             finished_growing = True
 
             if params("growing.reset_delta"):
@@ -217,7 +219,7 @@ def solve_main(objfun, x0, args, xl, xu, npt, rhobeg, rhoend, maxfun, nruns_so_f
 
         if not finished_growing:
             if do_logging:
-                logging.debug("Main loop: still growing (have %g of %g pts)" % (control.model.npt(), control.model.num_pts))
+                module_logger.debug("Main loop: still growing (have %g of %g pts)" % (control.model.npt(), control.model.num_pts))
 
         # Noise level exit check
         if finished_growing and params("noise.quit_on_noise_level") and control.all_values_within_noise_level(params):
@@ -247,7 +249,8 @@ def solve_main(objfun, x0, args, xl, xu, npt, rhobeg, rhoend, maxfun, nruns_so_f
                 min_sing_val=params("growing.full_rank.min_sing_val"),
                 sing_val_frac=params("growing.full_rank.svd_scale_factor"),
                 max_jac_cond=params("growing.full_rank.svd_max_jac_cond"),
-                get_chg_J=params("restarts.use_restarts") and params("restarts.auto_detect"))
+                get_chg_J=params("restarts.use_restarts") and params("restarts.auto_detect"),
+                throw_error_on_nans=params("interpolation.throw_error_on_nans"))
         if not interp_ok:
             if params("restarts.use_restarts") and params("restarts.use_soft_restarts"):
                 number_of_samples = max(nsamples(control.delta, control.rho, current_iter, nruns_so_far), 1)
@@ -270,9 +273,9 @@ def solve_main(objfun, x0, args, xl, xu, npt, rhobeg, rhoend, maxfun, nruns_so_f
 
 
         # Trust region step
-        d, gopt, H, gnew, crvmin = control.trust_region_step()
+        d, gopt, H, gnew, crvmin = control.trust_region_step(params)
         if do_logging:
-            logging.debug("Trust region step is d = " + str(d))
+            module_logger.debug("Trust region step is d = " + str(d))
         xnew = control.model.xopt() + d
         dnorm = min(LA.norm(d), control.delta)
 
@@ -288,7 +291,7 @@ def solve_main(objfun, x0, args, xl, xu, npt, rhobeg, rhoend, maxfun, nruns_so_f
 
         if dnorm < params("general.safety_step_thresh") * control.rho and not finished_growing and params("growing.safety.do_safety_step"):
             if do_logging:
-                logging.debug("Safety step during growing phase")
+                module_logger.debug("Safety step during growing phase")
 
             if params("logging.save_diagnostic_info"):
                 diagnostic_info.update_ratio(np.nan)
@@ -366,7 +369,7 @@ def solve_main(objfun, x0, args, xl, xu, npt, rhobeg, rhoend, maxfun, nruns_so_f
         elif dnorm < params("general.safety_step_thresh") * control.rho and finished_growing:
             # (start safety step)
             if do_logging:
-                logging.debug("Safety step (main phase)")
+                module_logger.debug("Safety step (main phase)")
 
             if params("logging.save_diagnostic_info"):
                 diagnostic_info.update_ratio(np.nan)
@@ -410,12 +413,12 @@ def solve_main(objfun, x0, args, xl, xu, npt, rhobeg, rhoend, maxfun, nruns_so_f
                 # Reduce rho
                 control.reduce_rho(current_iter, params)
                 if do_logging:
-                    logging.info("New rho = %g after %i function evaluations" % (control.rho, control.nf))
+                    module_logger.info("New rho = %g after %i function evaluations" % (control.rho, control.nf))
                     if control.n() < params("logging.n_to_print_whole_x_vector"):
-                        logging.debug("Best so far: f = %.15g at x = " % (control.model.fopt())
+                        module_logger.debug("Best so far: f = %.15g at x = " % (control.model.fopt())
                                       + str(control.model.xopt(abs_coordinates=True)))
                     else:
-                        logging.debug("Best so far: f = %.15g at x = [...]" % (control.model.fopt()))
+                        module_logger.debug("Best so far: f = %.15g at x = [...]" % (control.model.fopt()))
                 continue  # next iteration
             else:
                 # Quit on rho=rhoend
@@ -455,18 +458,18 @@ def solve_main(objfun, x0, args, xl, xu, npt, rhobeg, rhoend, maxfun, nruns_so_f
         else:
             # (start trust region step)
             if do_logging:
-                logging.debug("Standard trust region step")
+                module_logger.debug("Standard trust region step")
 
             # If growing, optionally perturb the trust region step in a new direction
             if not finished_growing and params("growing.perturb_trust_region_step"):
                 step_length = params('growing.delta_scale_new_dirns') * control.delta
                 dirn = control.get_new_direction_for_growing(step_length)
                 if do_logging:
-                    logging.debug("Perturbing trust region with step = %s" % str(dirn))
+                    module_logger.debug("Perturbing trust region with step = %s" % str(dirn))
                 d += dirn
                 xnew += dirn
                 if do_logging:
-                    logging.debug("New trust region step = %s" % str(d))
+                    module_logger.debug("New trust region step = %s" % str(d))
 
             # If finished growing, add chgJ and delta to restart auto-detect set
             if finished_growing and params("restarts.use_restarts") and params("restarts.auto_detect"):
@@ -513,6 +516,17 @@ def solve_main(objfun, x0, args, xl, xu, npt, rhobeg, rhoend, maxfun, nruns_so_f
             x = control.model.as_absolute_coordinates(xnew)
             number_of_samples = max(nsamples(control.delta, control.rho, current_iter, nruns_so_far), 1)
             rvec_list, f_list, num_samples_run, exit_info = control.evaluate_objective(x, number_of_samples, params)
+            if np.any(np.isnan(rvec_list)):
+                # Just exit without saving the current point
+                # We should be able to do a hard restart though, because it's unlikely 
+                # that we will get the same trust-region step after expanding the radius/re-initialising
+                module_logger.warning("NaN encountered in evaluation of trust-region step")
+                if params("interpolation.throw_error_on_nans"):
+                    raise np.linalg.LinAlgError("NaN encountered in objective evaluations")
+                
+                exit_info = ExitInformation(EXIT_EVAL_ERROR, "NaN received from objective function evaluation")
+                nruns_so_far += 1
+                break  # quit
             if exit_info is not None:
                 if num_samples_run > 0:
                     control.model.save_point(x, np.mean(rvec_list[:num_samples_run, :], axis=0), num_samples_run,
@@ -545,7 +559,7 @@ def solve_main(objfun, x0, args, xl, xu, npt, rhobeg, rhoend, maxfun, nruns_so_f
 
             # Update delta
             if do_logging:
-                logging.debug("Ratio = %g" % ratio)
+                module_logger.debug("Ratio = %g" % ratio)
             if params("logging.save_diagnostic_info"):
                 diagnostic_info.update_ratio(ratio)
                 diagnostic_info.update_slow_iter(-1)  # n/a, unless otherwise update
@@ -603,7 +617,7 @@ def solve_main(objfun, x0, args, xl, xu, npt, rhobeg, rhoend, maxfun, nruns_so_f
                 knew = control.model.npt()
 
             if do_logging:
-                logging.debug("Updating with knew = %i" % knew)
+                module_logger.debug("Updating with knew = %i" % knew)
             control.model.change_point(knew, xnew, rvec_list[0, :])  # expect step, not absolute x
             for i in range(1, num_samples_run):
                 control.model.add_new_sample(knew, rvec_extra=rvec_list[i, :])
@@ -615,7 +629,7 @@ def solve_main(objfun, x0, args, xl, xu, npt, rhobeg, rhoend, maxfun, nruns_so_f
                     diagnostic_info.update_slow_iter(1 if this_iter_slow else 0)
                 if finished_growing and should_terminate:
                     if do_logging:
-                        logging.info("Slow iteration  - terminating/restarting")
+                        module_logger.info("Slow iteration  - terminating/restarting")
                     if params("restarts.use_restarts") and params("restarts.use_soft_restarts"):
                         number_of_samples = max(nsamples(control.delta, control.rho, current_iter, nruns_so_far), 1)
                         exit_info = control.soft_restart(number_of_samples, nruns_so_far, params,
@@ -649,7 +663,7 @@ def solve_main(objfun, x0, args, xl, xu, npt, rhobeg, rhoend, maxfun, nruns_so_f
             # While growing, (optionally) add new directions
             if not finished_growing and params("growing.num_new_dirns_each_iter") > 0:
                 if do_logging:
-                    logging.debug("Still growing: adding %g new directions" % params("growing.num_new_dirns_each_iter"))
+                    module_logger.debug("Still growing: adding %g new directions" % params("growing.num_new_dirns_each_iter"))
                 number_of_samples = max(nsamples(control.delta, control.rho, current_iter, nruns_so_far), 1)
                 exit_info = control.add_new_direction_while_growing(number_of_samples, params)
                 if exit_info is not None:
@@ -682,7 +696,7 @@ def solve_main(objfun, x0, args, xl, xu, npt, rhobeg, rhoend, maxfun, nruns_so_f
                                        control.model.npt() - 1)  # cap at number of points
             if finished_growing and ratio > 0.0 and num_regression_steps > 0:
                 if do_logging:
-                    logging.info("Regression: moving %g points" % num_regression_steps)
+                    module_logger.info("Regression: moving %g points" % num_regression_steps)
                 number_of_samples = max(nsamples(control.delta, control.rho, current_iter, nruns_so_far), 1)
                 if params("regression.momentum_extra_steps"):  # move points as random extra steps
                     exit_info = control.move_furthest_points_momentum(d, number_of_samples, num_regression_steps, params)
@@ -729,7 +743,7 @@ def solve_main(objfun, x0, args, xl, xu, npt, rhobeg, rhoend, maxfun, nruns_so_f
                                                                                 np.log(np.maximum(restart_auto_detect_chgJ, 1e-15)))
 
                     if do_logging:
-                        logging.debug("Iter %g: (slope, intercept, r_value) = (%g, %g, %g)" % (current_iter, slope, intercept, r_value))
+                        module_logger.debug("Iter %g: (slope, intercept, r_value) = (%g, %g, %g)" % (current_iter, slope, intercept, r_value))
                     if slope > params("restarts.auto_detect.min_chgJ_slope") \
                             and r_value > params("restarts.auto_detect.min_correl"):
                         # increasing trend, with at least some positive correlation
@@ -740,9 +754,9 @@ def solve_main(objfun, x0, args, xl, xu, npt, rhobeg, rhoend, maxfun, nruns_so_f
                 # Data available (full NumPy vectors of fixed length): restart_auto_detect_delta, restart_auto_detect_chgJ
                 if do_restart and params("restarts.use_soft_restarts"):
                     if do_logging:
-                        logging.info("Auto detection: need to do a restart")
-                        logging.debug("delta history = %s" % str(restart_auto_detect_delta))
-                        logging.debug("chgJ history = %s" % str(restart_auto_detect_chgJ))
+                        module_logger.info("Auto detection: need to do a restart")
+                        module_logger.debug("delta history = %s" % str(restart_auto_detect_delta))
+                        module_logger.debug("chgJ history = %s" % str(restart_auto_detect_chgJ))
                     number_of_samples = max(nsamples(control.delta, control.rho, current_iter, nruns_so_far), 1)
                     exit_info = control.soft_restart(number_of_samples, nruns_so_far, params,
                                                      x_in_abs_coords_to_save=None, rvec_to_save=None,
@@ -759,7 +773,7 @@ def solve_main(objfun, x0, args, xl, xu, npt, rhobeg, rhoend, maxfun, nruns_so_f
                     continue  # next iteration
                 elif do_restart:
                     if do_logging:
-                        logging.info("Auto detection: need to do a restart")
+                        module_logger.info("Auto detection: need to do a restart")
                     exit_info = ExitInformation(EXIT_AUTO_DETECT_RESTART_WARNING, "Auto-detected restart")
                     nruns_so_far += 1
                     break  # quit
@@ -767,7 +781,7 @@ def solve_main(objfun, x0, args, xl, xu, npt, rhobeg, rhoend, maxfun, nruns_so_f
 
             # Otherwise (ratio < eta1 = 0.1), check & fix geometry
             if do_logging:
-                logging.debug("Checking and possibly improving geometry (unsuccessful step)")
+                module_logger.debug("Checking and possibly improving geometry (unsuccessful step)")
             distsq = max((2.0 * control.delta) ** 2, (10.0 * control.rho) ** 2)
             update_delta = False
             number_of_samples = max(nsamples(control.delta, control.rho, current_iter, nruns_so_far), 1)
@@ -812,12 +826,12 @@ def solve_main(objfun, x0, args, xl, xu, npt, rhobeg, rhoend, maxfun, nruns_so_f
                 # Reduce rho
                 control.reduce_rho(current_iter, params)
                 if do_logging:
-                    logging.info("New rho = %g after %i function evaluations" % (control.rho, control.nf))
+                    module_logger.info("New rho = %g after %i function evaluations" % (control.rho, control.nf))
                     if control.n() < params("logging.n_to_print_whole_x_vector"):
-                        logging.debug("Best so far: f = %.15g at x = " % (control.model.fopt())
+                        module_logger.debug("Best so far: f = %.15g at x = " % (control.model.fopt())
                                       + str(control.model.xopt(abs_coordinates=True)))
                     else:
-                        logging.debug("Best so far: f = %.15g at x = [...]" % (control.model.fopt()))
+                        module_logger.debug("Best so far: f = %.15g at x = [...]" % (control.model.fopt()))
                 continue  # next iteration
             else:
                 # Quit on rho=rhoend
@@ -845,14 +859,14 @@ def solve_main(objfun, x0, args, xl, xu, npt, rhobeg, rhoend, maxfun, nruns_so_f
     # Quit & return the important information
     x, rvec, f, jacmin, nsamples = control.model.get_final_results()
     if do_logging:
-        logging.debug("At return from DFO-LS, number of function evals = %i" % nf)
-        logging.debug("Smallest objective value = %.15g at x = " % f + str(x))
+        module_logger.debug("At return from DFO-LS, number of function evals = %i" % nf)
+        module_logger.debug("Smallest objective value = %.15g at x = " % f + str(x))
     return x, rvec, f, jacmin, nsamples, control.nf, control.nx, nruns_so_far, exit_info, diagnostic_info
 
 
-def solve(objfun, x0, args=(), bounds=None, npt=None, rhobeg=None, rhoend=1e-8, maxfun=None, nsamples=None, user_params=None,
+def solve(objfun, x0, args=(), bounds=None, projections=[], npt=None, rhobeg=None, rhoend=1e-8, maxfun=None, nsamples=None, user_params=None,
           objfun_has_noise=False, scaling_within_bounds=False, do_logging=True, print_progress=False):
-    x0 = x0.astype(np.float)
+    x0 = x0.astype(float)
     n = len(x0)
 
     # Set missing inputs (if not specified) to some sensible defaults
@@ -861,13 +875,17 @@ def solve(objfun, x0, args=(), bounds=None, npt=None, rhobeg=None, rhoend=1e-8,
         xu = None
     else:
         assert len(bounds) == 2, "bounds must be a 2-tuple of (lower, upper), where both are arrays of size(x0)"
-        xl = bounds[0].astype(np.float) if bounds[0] is not None else None
-        xu = bounds[1].astype(np.float) if bounds[1] is not None else None
+        xl = bounds[0].astype(float) if bounds[0] is not None else None
+        xu = bounds[1].astype(float) if bounds[1] is not None else None
 
     if (xl is None or xu is None) and scaling_within_bounds:
         scaling_within_bounds = False
         warnings.warn("Ignoring scaling_within_bounds=True for unconstrained problem/1-sided bounds", RuntimeWarning)
 
+    if projections and scaling_within_bounds:
+        scaling_within_bounds = False
+        warnings.warn("Ignoring scaling_within_bounds=True for problem with arbitrary constraints", RuntimeWarning)
+
     if xl is None:
         xl = -1e20 * np.ones((n,))  # unconstrained
     if xu is None:
@@ -881,6 +899,18 @@ def solve(objfun, x0, args=(), bounds=None, npt=None, rhobeg=None, rhoend=1e-8,
     if nsamples is None:
         nsamples = lambda delta, rho, iter, nruns: 1  # no averaging
 
+    # If using arbitrary constraints, create projection from bounds
+    if projections:
+        xlb = xl.copy()
+        xub = xu.copy()
+        bproj = lambda w: pbox(w,xlb,xub)
+        projections = list(projections)
+        projections.append(bproj)
+
+        # since using arbitrary constraints, don't constrain otherwise
+        xl = -1e20 * np.ones((n,))
+        xu = 1e20 * np.ones((n,))
+
     # Set parameters
     params = ParameterList(int(n), int(npt), int(maxfun), objfun_has_noise=objfun_has_noise)  # make sure int, not np.int
     if user_params is not None:
@@ -975,6 +1005,13 @@ def solve(objfun, x0, args=(), bounds=None, npt=None, rhobeg=None, rhoend=1e-8,
         results = OptimResults(None, None, None, None, 0, 0, 0, exit_flag, exit_msg)
         return results
 
+    # Enforce arbitrary constraint bounds on x0
+    if projections:
+        xp = dykstra(projections,x0,max_iter=params("dykstra.max_iters"),tol=params("dykstra.d_tol"))
+        if not np.allclose(xp,x0):
+            warnings.warn("x0 not feasible w.r.t given constraints, adjusting", RuntimeWarning)
+            x0 = xp.copy()
+
     # Enforce lower & upper bounds on x0
     idx = (x0 <= xl)
     if np.any(idx):
@@ -992,7 +1029,7 @@ def solve(objfun, x0, args=(), bounds=None, npt=None, rhobeg=None, rhoend=1e-8,
     nf = 0
     nx = 0
     xmin, rmin, fmin, jacmin, nsamples_min, nf, nx, nruns, exit_info, diagnostic_info = \
-        solve_main(objfun, x0, args, xl, xu, npt, rhobeg, rhoend, maxfun, nruns, nf, nx, nsamples, params,
+        solve_main(objfun, x0, args, xl, xu, projections, npt, rhobeg, rhoend, maxfun, nruns, nf, nx, nsamples, params,
                     diagnostic_info, scaling_changes, default_growing_method_set_by_user=default_growing_method_set_by_user,
                    do_logging=do_logging, print_progress=print_progress)
 
@@ -1007,28 +1044,28 @@ def solve(objfun, x0, args=(), bounds=None, npt=None, rhobeg=None, rhoend=1e-8,
             npt = min(npt, params("restarts.max_npt"))
 
         if do_logging:
-            logging.info("Restarting from finish point (f = %g) after %g function evals; using rhobeg = %g and rhoend = %g"
+            module_logger.info("Restarting from finish point (f = %g) after %g function evals; using rhobeg = %g and rhoend = %g"
                      % (fmin, nf, rhobeg, rhoend))
         if params("restarts.hard.use_old_rk"):
             xmin2, rmin2, fmin2, jacmin2, nsamples2, nf, nx, nruns, exit_info, diagnostic_info = \
-                solve_main(objfun, xmin, args, xl, xu, npt, rhobeg, rhoend, maxfun, nruns, nf, nx, nsamples, params,
+                solve_main(objfun, xmin, args, xl, xu, projections, npt, rhobeg, rhoend, maxfun, nruns, nf, nx, nsamples, params,
                             diagnostic_info, scaling_changes, r0_avg_old=rmin, r0_nsamples_old=nsamples_min,
                            do_logging=do_logging, print_progress=print_progress)
         else:
             xmin2, rmin2, fmin2, jacmin2, nsamples2, nf, nx, nruns, exit_info, diagnostic_info = \
-                solve_main(objfun, xmin, args, xl, xu, npt, rhobeg, rhoend, maxfun, nruns, nf, nx, nsamples, params,
+                solve_main(objfun, xmin, args, xl, xu, projections, npt, rhobeg, rhoend, maxfun, nruns, nf, nx, nsamples, params,
                            diagnostic_info, scaling_changes, do_logging=do_logging, print_progress=print_progress)
 
         if fmin2 < fmin or np.isnan(fmin):
             if do_logging:
-                logging.info("Successful run with new f = %s compared to old f = %s" % (fmin2, fmin))
+                module_logger.info("Successful run with new f = %s compared to old f = %s" % (fmin2, fmin))
             last_successful_run = nruns
             (xmin, rmin, fmin, nsamples_min) = (xmin2, rmin2, fmin2, nsamples2)
             if jacmin2 is not None:  # may be None if finished during setup phase, in which case just use old Jacobian
                 jacmin = jacmin2
         else:
             if do_logging:
-                logging.info("Unsuccessful run with new f = %s compared to old f = %s" % (fmin2, fmin))
+                module_logger.info("Unsuccessful run with new f = %s compared to old f = %s" % (fmin2, fmin))
 
     if nruns - last_successful_run >= params("restarts.max_unsuccessful_restarts"):
         exit_info = ExitInformation(EXIT_SUCCESS, "Reached maximum number of unsuccessful restarts")
@@ -1046,7 +1083,7 @@ def solve(objfun, x0, args=(), bounds=None, npt=None, rhobeg=None, rhoend=1e-8,
         results.diagnostic_info = df
 
     if do_logging:
-        logging.info("Did a total of %g run(s)" % nruns)
+        module_logger.info("Did a total of %g run(s)" % nruns)
 
     return results
 

dfols/trust_region.py

@@ -11,6 +11,15 @@ produces a new vector d which (approximately) solves the trust region subproblem
 The other outputs: gnew is the gradient of the model at d, and crvmin has
 information about the curvature of the model at the solution.
 
+For handling arbitrary constraints, the call is
+    d, gnew, crvmin = ctrsbox(xopt, g, H, projections, delta)
+which produces a new vector d approximately solving the constrained trust region subproblem:
+    min_{d}  g'*d + 0.5*d'*H*d
+    s.t.    ||d|| <= delta
+            xopt + d is feasible w.r.t. the constraint set C
+The other outputs: gnew is the gradient of the model at d, and crvmin has
+information about the curvature of the model at the solution.
+
 We also provide a function for maximising the absolute value of a linear function
 inside a similar trust region - this is useful for geometry steps.
 The call
@@ -23,6 +32,13 @@ With this value, the variable d=x-xbase solves the problem
     min_s  abs(c + g' * d)
     s.t.   lower <= xbase + d <= upper
           ||d|| <= delta
+Again, we have a version of this for handling arbitrary constraints
+The call
+    x = ctrsbox_geometry(xbase, c, g, projections, Delta)
+Solves
+    min_s  abs(c + g' * d)
+    s.t.   xbase + d is feasible w.r.t. the constraint set C
+          ||d|| <= delta
 
 Notes
 ----
@@ -63,13 +79,77 @@ except ImportError:
     # Fall back to Python implementation
     USE_FORTRAN = False
 
+from .util import dykstra, pball, pbox, sumsq, model_value
 
-from .util import sumsq
+__all__ = ['ctrsbox', 'ctrsbox_geometry', 'trsbox', 'trsbox_geometry']
 
+ZERO_THRESH = 1e-14
 
-__all__ = ['trsbox', 'trsbox_geometry']
+def ctrsbox(xopt, g, H, projections, delta, d_max_iters=100, d_tol=1e-10, use_fortran=USE_FORTRAN):
+    n = xopt.size
+    assert xopt.shape == (n,), "xopt has wrong shape (should be vector)"
+    assert g.shape == (n,), "g and xopt have incompatible sizes"
+    assert len(H.shape) == 2, "H must be a matrix"
+    assert H.shape == (n,n), "H and xopt have incompatible sizes"
+    assert np.allclose(H, H.T), "H must be symmetric"
+    assert delta > 0.0, "delta must be strictly positive"
 
-ZERO_THRESH = 1e-14
+    d = np.zeros((n,))
+    gnew = g.copy()
+    gy = g.copy()
+    crvmin = -1.0
+    y = d.copy()
+    eta = 1.2 # L backtrack scaling factor
+    t = 1
+
+    # Initial guess of L is norm(Hessian)
+    L = np.linalg.norm(H, 2)
+
+    # trust region is a ball of radius delta around xopt
+    trproj = lambda w: pball(w, xopt, delta)
+
+    # combine trust region constraints with user-entered constraints
+    P = list(projections)  # make a copy of the projections list
+    P.append(trproj)
+    def proj(d0):
+        p = dykstra(P, xopt+d0, max_iter=d_max_iters, tol=d_tol)
+        # we want the step only, so we subtract xopt
+        # from the new point: proj(xk+d) - xk
+        return p - xopt
+
+    MAX_LOOP_ITERS = 100 * n ** 2
+
+    # projected GD loop 
+    for ii in range(MAX_LOOP_ITERS):
+        w = y - (1/L)*gy
+        prev_d = d.copy()
+        d = proj(w)
+
+        # size of step taken
+        s = d - prev_d
+        stplen = np.linalg.norm(s)
+
+        # update true gradient
+        gnew += H.dot(s)
+
+        # update CRVMIN
+        crv = s.dot(H).dot(s)/sumsq(s) if sumsq(s) >= ZERO_THRESH else crvmin
+        crvmin = min(crvmin, crv) if crvmin != -1.0 else crv
+
+        # exit condition
+        if stplen <= ZERO_THRESH:
+            break
+
+        # momentum update
+        prev_t = t
+        t = (1 + np.sqrt(1 + 4 * t ** 2))/2
+        prev_y = y.copy()
+        y = d + s*(prev_t - 1)/t
+
+        # update gradient w.r.t y
+        gy += H.dot(y - prev_y)
+
+    return d, gnew, crvmin
 
 
 def trsbox(xopt, g, H, sl, su, delta, use_fortran=USE_FORTRAN):
@@ -103,7 +183,7 @@ def trsbox(xopt, g, H, sl, su, delta, use_fortran=USE_FORTRAN):
     iterc = 0
     nact = 0  # number of fixed variables
 
-    xbdi = np.zeros((n,), dtype=np.int)  # fix x_i at bounds? [values -1, 0, 1]
+    xbdi = np.zeros((n,), dtype=int)  # fix x_i at bounds? [values -1, 0, 1]
     xbdi[(xopt <= sl) & (g >= 0.0)] = -1
     xbdi[(xopt >= su) & (g <= 0.0)] = 1
 
@@ -405,8 +485,63 @@ def ball_step(x0, g, Delta):
     if sqrt(gsqnorm) < ZERO_THRESH:  # Error catching: if g=0, make no step
         return 0.0
     else:
-        return (sqrt(gdotx0**2 + gsqnorm*(Delta**2 - x0sqnorm)) - gdotx0) / gsqnorm
+        # Sqrt had negative input on prob 46 in OG DFOLS with noise
+        #  print("Inside of the sqrt:", gdotx0**2 + gsqnorm*(Delta**2 - x0sqnorm))
+        # Got Inside of the sqrt: -3.608971127647144e-42
+        # Added max(0,...) here
+        return (sqrt(np.maximum(0,gdotx0**2 + gsqnorm*(Delta**2 - x0sqnorm))) - gdotx0) / gsqnorm
+
+def ctrsbox_linear(xbase, g, projections, Delta, d_max_iters=100, d_tol=1e-10, use_fortran=USE_FORTRAN):
+    # Solve the convex program:
+    #   min_d   g' * d
+    #   s.t.    xbase + d is feasible w.r.t. constraint set C
+    #           ||d||^2 <= Delta^2
+
+    n = g.size
+    d = np.zeros((n,))
+    y = d.copy()
+    t = 1
+    dirn = -g
+    cons_dirns = []
+
+    # If g[i] = 0, never step along this direction
+    constant_directions = np.where(np.abs(dirn) < ZERO_THRESH)[0]
+    dirn[constant_directions] = 0.0
+
+    # trust region is a ball of radius delta centered around xbase
+    trproj = lambda w: pball(w, xbase, Delta)
 
+    # combine trust region constraints with user-entered constraints
+    P = list(projections)  # make a copy of the projections list
+    P.append(trproj)
+    def proj(d0):
+        p = dykstra(P, xbase + d0, max_iter=d_max_iters, tol=d_tol)
+        # we want the step only, so we subtract
+        # xbase from the new point: proj(xk + d) - xk
+        return p - xbase
+
+    MAX_LOOP_ITERS = 100 * n ** 2
+
+    # projected GD loop 
+    for ii in range(MAX_LOOP_ITERS):
+        w = y + dirn
+        prev_d = d.copy()
+        d = proj(w)
+
+        s = d - prev_d
+        stplen = np.linalg.norm(s)
+
+        # exit condition
+        if stplen <= ZERO_THRESH:
+            break
+
+        # 'momentum' update
+        prev_t = t
+        t = (1 + np.sqrt(1 + 4 * t ** 2))/2
+        prev_y = y.copy()
+        y = d + s*(prev_t - 1)/t
+
+    return d
 
 def trsbox_linear(g, a_in, b_in, Delta, use_fortran=USE_FORTRAN):
     # Solve the convex program:
@@ -466,6 +601,22 @@ def trsbox_linear(g, a_in, b_in, Delta, use_fortran=USE_FORTRAN):
             dirn[idx_hit] = 0.0  # no more searching this direction
     return x
 
+def ctrsbox_geometry(xbase, c, g, projections, Delta, d_max_iters=100, d_tol=1e-10, use_fortran=USE_FORTRAN):
+    # Given a Lagrange polynomial defined by: L(x) = c + g' * (x - xbase)
+    # Maximise |L(x)| in a box + trust region - that is, solve:
+    #   max_x  abs(c + g' * (x - xbase))
+    #    s.t.  x is feasible w.r.t constraint set C
+    #          ||x-xbase|| <= Delta
+    # Setting s = x-xbase (or x = xbase + s), this is equivalent to:
+    #   max_s  abs(c + g' * s)
+    #   s.t.   xbase + s is is feasible w.r.t constraint set C
+    #          ||s|| <= Delta
+    smin = ctrsbox_linear(xbase, g, projections, Delta, d_max_iters=100, d_tol=1e-10, use_fortran=use_fortran)  # minimise g' * s
+    smax = ctrsbox_linear(xbase, -g, projections, Delta, d_max_iters=100, d_tol=1e-10, use_fortran=use_fortran)  # maximise g' * s
+    if abs(c + np.dot(g, smin)) >= abs(c + np.dot(g, smax)):  # choose the one with largest absolute value
+        return smin
+    else:
+        return smax
 
 def trsbox_geometry(xbase, c, g, lower, upper, Delta, use_fortran=USE_FORTRAN):
     # Given a Lagrange polynomial defined by: L(x) = c + g' * (x - xbase)