DFO-LS 1.2.1__py3-none-any.whl → 1.5.0__py3-none-any.whl

dfols/solver.py

@@ -43,13 +43,15 @@ from .util import *
 
 __all__ = ['solve']
 
+module_logger = logging.getLogger(__name__) 
+
 
 # A container for the results of the optimization routine
 class OptimResults(object):
-    def __init__(self, xmin, rmin, fmin, jacmin, nf, nx, nruns, exit_flag, exit_msg):
+    def __init__(self, xmin, rmin, objmin, jacmin, nf, nx, nruns, exit_flag, exit_msg):
         self.x = xmin
         self.resid = rmin
-        self.f = fmin
+        self.obj = objmin
         self.jacobian = jacmin
         self.nf = nf
         self.nx = nx
@@ -75,7 +77,7 @@ class OptimResults(object):
                 output += "Residual vector = %s\n" % str(self.resid)
             else:
                 output += "Not showing residual vector because it is too long; check self.resid\n"
-            output += "Objective value f(xmin) = %.10g\n" % self.f
+            output += "Objective value f(xmin) = %.10g\n" % self.obj
             output += "Needed %g objective evaluations (at %g points)\n" % (self.nf, self.nx)
             if self.nruns > 1:
                 output += "Did a total of %g runs\n" % self.nruns
@@ -93,8 +95,8 @@ class OptimResults(object):
         return output
 
 
-def solve_main(objfun, x0, args, xl, xu, npt, rhobeg, rhoend, maxfun, nruns_so_far, nf_so_far, nx_so_far, nsamples, params,
-               diagnostic_info, scaling_changes, r0_avg_old=None, r0_nsamples_old=None, default_growing_method_set_by_user=None,
+def solve_main(objfun, x0, argsf, xl, xu, projections, npt, rhobeg, rhoend, maxfun, nruns_so_far, nf_so_far, nx_so_far, nsamples, params,
+               diagnostic_info, scaling_changes, h=None, lh=None, argsh=(), prox_uh=None, argsprox=None, r0_avg_old=None, r0_nsamples_old=None, default_growing_method_set_by_user=None,
                do_logging=True, print_progress=False):
     # Evaluate at x0 (keep nf, nx correct and check for f < 1e-12)
     # The hard bit is determining what m = len(r0) should be, and allocating memory appropriately
@@ -103,18 +105,17 @@ def solve_main(objfun, x0, args, xl, xu, npt, rhobeg, rhoend, maxfun, nruns_so_f
         # Evaluate the first time...
         nf = nf_so_far + 1
         nx = nx_so_far + 1
-        r0, f0 = eval_least_squares_objective(objfun, remove_scaling(x0, scaling_changes),
-                                              args=args, eval_num=nf, pt_num=nx,
+        r0, obj0 = eval_least_squares_with_regularisation(objfun, remove_scaling(x0, scaling_changes), h, 
+                                              argsf=argsf, argsh=argsh, verbose=do_logging, eval_num=nf, pt_num=nx,
                                               full_x_thresh=params("logging.n_to_print_whole_x_vector"),
-                                              check_for_overflow=params("general.check_objfun_for_overflow"),
-                                              verbose=do_logging)
+                                              check_for_overflow=params("general.check_objfun_for_overflow"))
         m = len(r0)
 
         # Now we have m, we can evaluate the rest of the times
         rvec_list = np.zeros((number_of_samples, m))
-        f_list = np.zeros((number_of_samples,))
+        obj_list = np.zeros((number_of_samples,))
         rvec_list[0, :] = r0
-        f_list[0] = f0
+        obj_list[0] = obj0
         num_samples_run = 1
         exit_info = None
 
@@ -126,15 +127,20 @@ def solve_main(objfun, x0, args, xl, xu, npt, rhobeg, rhoend, maxfun, nruns_so_f
 
             nf += 1
             # Don't increment nx for x0 - we did this earlier
-            rvec_list[i, :], f_list[i] = eval_least_squares_objective(objfun, remove_scaling(x0, scaling_changes), args=args, eval_num=nf, pt_num=nx,
+            rvec_list[i, :], obj_list[i] = eval_least_squares_with_regularisation(objfun, remove_scaling(x0, scaling_changes), h, 
+                                                argsf=argsf, argsh=argsh, verbose=do_logging, eval_num=nf, pt_num=nx,
                                                 full_x_thresh=params("logging.n_to_print_whole_x_vector"),
-                                                check_for_overflow=params("general.check_objfun_for_overflow"),
-                                                verbose=do_logging)
+                                                check_for_overflow=params("general.check_objfun_for_overflow"))
             num_samples_run += 1
 
         r0_avg = np.mean(rvec_list[:num_samples_run, :], axis=0)
-        if sumsq(r0_avg) <= params("model.abs_tol"):
-            exit_info = ExitInformation(EXIT_SUCCESS, "Objective is sufficiently small")
+        # NOTE: modify objvalue here
+        if h is None:
+            if sumsq(r0_avg) <= params("model.abs_tol"):
+                exit_info = ExitInformation(EXIT_SUCCESS, "Objective is sufficiently small")
+        else:
+            if sumsq(r0_avg) + h(remove_scaling(x0, scaling_changes), *argsh)<= params("model.abs_tol"):
+                exit_info = ExitInformation(EXIT_SUCCESS, "Objective is sufficiently small")
 
         if exit_info is not None:
             return x0, r0_avg, sumsq(r0_avg), None, num_samples_run, nf, nx, nruns_so_far+1, exit_info, diagnostic_info
@@ -153,15 +159,15 @@ def solve_main(objfun, x0, args, xl, xu, npt, rhobeg, rhoend, maxfun, nruns_so_f
         # However, this can fail for m<n, so need to use an alternative method (perturb_trust_region_step)
         if m < len(x0):
             if do_logging:
-                logging.debug("Inverse problem (m<n), switching default growing method")
+                module_logger.debug("Inverse problem (m<n), switching default growing method")
             params('growing.full_rank.use_full_rank_interp', new_value=False)
             params('growing.perturb_trust_region_step', new_value=True)
             if not params.params_changed['growing.delta_scale_new_dirns']:
                 params('growing.delta_scale_new_dirns', new_value=0.1)
 
     # Initialise controller
-    control = Controller(objfun, args, x0, r0_avg, num_samples_run, xl, xu, npt, rhobeg, rhoend, nf, nx, maxfun,
-                         params, scaling_changes, do_logging)
+    control = Controller(objfun, argsf, x0, r0_avg, num_samples_run, xl, xu, projections, npt, rhobeg, rhoend, nf, nx, maxfun,
+                         params, scaling_changes, do_logging, h=h, lh=lh, argsh=argsh,  prox_uh=prox_uh, argsprox=argsprox)
 
     # Initialise interpolation set
     number_of_samples = max(nsamples(control.delta, control.rho, 0, nruns_so_far), 1)
@@ -169,15 +175,15 @@ def solve_main(objfun, x0, args, xl, xu, npt, rhobeg, rhoend, maxfun, nruns_so_f
                          npt - 1)  # cap at npt
     if params("init.random_initial_directions"):
         if do_logging:
-            logging.info("Initialising (random directions)")
+            module_logger.info("Initialising (random directions)")
         exit_info = control.initialise_random_directions(number_of_samples, num_directions, params)
     else:
         if do_logging:
-            logging.info("Initialising (coordinate directions)")
+            module_logger.info("Initialising (coordinate directions)")
         exit_info = control.initialise_coordinate_directions(number_of_samples, num_directions, params)
     if exit_info is not None:
-        x, rvec, f, jacmin, nsamples = control.model.get_final_results()
-        return x, rvec, f, None, nsamples, control.nf, control.nx, nruns_so_far + 1, exit_info, diagnostic_info
+        x, rvec, obj, jacmin, nsamples = control.model.get_final_results()
+        return x, rvec, obj, None, nsamples, control.nf, control.nx, nruns_so_far + 1, exit_info, diagnostic_info
 
     finished_growing = (control.model.npt() >= control.model.num_pts)  # have we finished growing the initial set yet?
 
@@ -195,18 +201,18 @@ def solve_main(objfun, x0, args, xl, xu, npt, rhobeg, rhoend, maxfun, nruns_so_f
     # ------------------------------------------
     current_iter = -1
     if do_logging:
-        logging.info("Beginning main loop")
+        module_logger.info("Beginning main loop")
     if print_progress:
         print("{:^5}{:^7}{:^10}{:^10}{:^10}{:^10}{:^7}".format("Run", "Iter", "Obj", "Grad", "Delta", "rho", "Evals"))
     while True:
         current_iter += 1
 
         if do_logging:
-            logging.debug("*** Iter %g (delta = %g, rho = %g) ***" % (current_iter, control.delta, control.rho))
+            module_logger.debug("*** Iter %g (delta = %g, rho = %g) ***" % (current_iter, control.delta, control.rho))
 
         if (not finished_growing) and control.model.npt() >= control.model.num_pts:
             if do_logging:
-                logging.info("Finished growing init set")
+                module_logger.info("Finished growing init set")
             finished_growing = True
 
             if params("growing.reset_delta"):
@@ -217,7 +223,7 @@ def solve_main(objfun, x0, args, xl, xu, npt, rhobeg, rhoend, maxfun, nruns_so_f
 
         if not finished_growing:
             if do_logging:
-                logging.debug("Main loop: still growing (have %g of %g pts)" % (control.model.npt(), control.model.num_pts))
+                module_logger.debug("Main loop: still growing (have %g of %g pts)" % (control.model.npt(), control.model.num_pts))
 
         # Noise level exit check
         if finished_growing and params("noise.quit_on_noise_level") and control.all_values_within_noise_level(params):
@@ -247,7 +253,8 @@ def solve_main(objfun, x0, args, xl, xu, npt, rhobeg, rhoend, maxfun, nruns_so_f
                 min_sing_val=params("growing.full_rank.min_sing_val"),
                 sing_val_frac=params("growing.full_rank.svd_scale_factor"),
                 max_jac_cond=params("growing.full_rank.svd_max_jac_cond"),
-                get_chg_J=params("restarts.use_restarts") and params("restarts.auto_detect"))
+                get_chg_J=params("restarts.use_restarts") and params("restarts.auto_detect"),
+                throw_error_on_nans=params("interpolation.throw_error_on_nans"))
         if not interp_ok:
             if params("restarts.use_restarts") and params("restarts.use_soft_restarts"):
                 number_of_samples = max(nsamples(control.delta, control.rho, current_iter, nruns_so_far), 1)
@@ -268,16 +275,30 @@ def solve_main(objfun, x0, args, xl, xu, npt, rhobeg, rhoend, maxfun, nruns_so_f
                 nruns_so_far += 1
                 break  # quit
 
-
-        # Trust region step
-        d, gopt, H, gnew, crvmin = control.trust_region_step()
+        tau = 1.0 # ratio used in the safety phase
+        if h is None:
+            # Trust region step
+            d, gopt, H, gnew, crvmin = control.trust_region_step(params)
+        else:
+            # Calculate criticality measure
+            criticality_measure = control.evaluate_criticality_measure(params)
+            # Trust region step
+            d, gopt, H, gnew, crvmin = control.trust_region_step(params, criticality_measure)
+            try:
+                tau = min(criticality_measure/(LA.norm(gopt)+lh), 1.0)
+            except ValueError:
+                # In some instances, gopt can have nan/inf values -- this ultimately calls a safety step and is generally fine
+                # but we need to set a value for tau nonetheless
+                tau = 1.0
+        
         if do_logging:
-            logging.debug("Trust region step is d = " + str(d))
+            module_logger.debug("Trust region step is d = " + str(d))
+        
         xnew = control.model.xopt() + d
         dnorm = min(LA.norm(d), control.delta)
 
         if print_progress:
-            print("{:^5}{:^7}{:^10.2e}{:^10.2e}{:^10.2e}{:^10.2e}{:^7}".format(nruns_so_far+1, current_iter+1, control.model.fopt(), np.linalg.norm(gopt), control.delta, control.rho, control.nf))
+            print("{:^5}{:^7}{:^10.2e}{:^10.2e}{:^10.2e}{:^10.2e}{:^7}".format(nruns_so_far+1, current_iter+1, control.model.objopt(), np.linalg.norm(gopt), control.delta, control.rho, control.nf))
 
         if params("logging.save_diagnostic_info"):
             diagnostic_info.save_info_from_control(control, nruns_so_far, current_iter,
@@ -286,9 +307,9 @@ def solve_main(objfun, x0, args, xl, xu, npt, rhobeg, rhoend, maxfun, nruns_so_f
             diagnostic_info.update_interpolation_information(interp_error, ls_interp_cond_num, linalg_resid,
                                                              sqrt(norm_J_error), LA.norm(gopt), LA.norm(d))
 
-        if dnorm < params("general.safety_step_thresh") * control.rho and not finished_growing and params("growing.safety.do_safety_step"):
+        if dnorm < tau * params("general.safety_step_thresh") * control.rho and not finished_growing and params("growing.safety.do_safety_step"):
             if do_logging:
-                logging.debug("Safety step during growing phase")
+                module_logger.debug("Safety step during growing phase")
 
             if params("logging.save_diagnostic_info"):
                 diagnostic_info.update_ratio(np.nan)
@@ -366,7 +387,7 @@ def solve_main(objfun, x0, args, xl, xu, npt, rhobeg, rhoend, maxfun, nruns_so_f
         elif dnorm < params("general.safety_step_thresh") * control.rho and finished_growing:
             # (start safety step)
             if do_logging:
-                logging.debug("Safety step (main phase)")
+                module_logger.debug("Safety step (main phase)")
 
             if params("logging.save_diagnostic_info"):
                 diagnostic_info.update_ratio(np.nan)
@@ -410,12 +431,12 @@ def solve_main(objfun, x0, args, xl, xu, npt, rhobeg, rhoend, maxfun, nruns_so_f
                 # Reduce rho
                 control.reduce_rho(current_iter, params)
                 if do_logging:
-                    logging.info("New rho = %g after %i function evaluations" % (control.rho, control.nf))
+                    module_logger.info("New rho = %g after %i function evaluations" % (control.rho, control.nf))
                     if control.n() < params("logging.n_to_print_whole_x_vector"):
-                        logging.debug("Best so far: f = %.15g at x = " % (control.model.fopt())
+                        module_logger.debug("Best so far: f = %.15g at x = " % (control.model.objopt())
                                       + str(control.model.xopt(abs_coordinates=True)))
                     else:
-                        logging.debug("Best so far: f = %.15g at x = [...]" % (control.model.fopt()))
+                        module_logger.debug("Best so far: f = %.15g at x = [...]" % (control.model.objopt()))
                 continue  # next iteration
             else:
                 # Quit on rho=rhoend
@@ -436,8 +457,9 @@ def solve_main(objfun, x0, args, xl, xu, npt, rhobeg, rhoend, maxfun, nruns_so_f
                 else:
                     # Cannot reduce rho, so check xnew and quit
                     x = control.model.as_absolute_coordinates(xnew)
+                    ##print("x from xnew", x)
                     number_of_samples = max(nsamples(control.delta, control.rho, current_iter, nruns_so_far), 1)
-                    rvec_list, f_list, num_samples_run, exit_info = control.evaluate_objective(x, number_of_samples,
+                    rvec_list, obj_list, num_samples_run, exit_info = control.evaluate_objective(x, number_of_samples,
                                                                                                params)
                     
                     if num_samples_run > 0:
@@ -455,18 +477,18 @@ def solve_main(objfun, x0, args, xl, xu, npt, rhobeg, rhoend, maxfun, nruns_so_f
         else:
             # (start trust region step)
             if do_logging:
-                logging.debug("Standard trust region step")
+                module_logger.debug("Standard trust region step")
 
             # If growing, optionally perturb the trust region step in a new direction
             if not finished_growing and params("growing.perturb_trust_region_step"):
                 step_length = params('growing.delta_scale_new_dirns') * control.delta
                 dirn = control.get_new_direction_for_growing(step_length)
                 if do_logging:
-                    logging.debug("Perturbing trust region with step = %s" % str(dirn))
+                    module_logger.debug("Perturbing trust region with step = %s" % str(dirn))
                 d += dirn
                 xnew += dirn
                 if do_logging:
-                    logging.debug("New trust region step = %s" % str(d))
+                    module_logger.debug("New trust region step = %s" % str(d))
 
             # If finished growing, add chgJ and delta to restart auto-detect set
             if finished_growing and params("restarts.use_restarts") and params("restarts.auto_detect"):
@@ -511,8 +533,20 @@ def solve_main(objfun, x0, args, xl, xu, npt, rhobeg, rhoend, maxfun, nruns_so_f
 
             # Evaluate new point
             x = control.model.as_absolute_coordinates(xnew)
+            ##print("x from xnew again", x)
             number_of_samples = max(nsamples(control.delta, control.rho, current_iter, nruns_so_far), 1)
-            rvec_list, f_list, num_samples_run, exit_info = control.evaluate_objective(x, number_of_samples, params)
+            rvec_list, obj_list, num_samples_run, exit_info = control.evaluate_objective(x, number_of_samples, params)
+            if np.any(np.isnan(rvec_list)):
+                # Just exit without saving the current point
+                # We should be able to do a hard restart though, because it's unlikely 
+                # that we will get the same trust-region step after expanding the radius/re-initialising
+                module_logger.warning("NaN encountered in evaluation of trust-region step")
+                if params("interpolation.throw_error_on_nans"):
+                    raise np.linalg.LinAlgError("NaN encountered in objective evaluations")
+                
+                exit_info = ExitInformation(EXIT_EVAL_ERROR, "NaN received from objective function evaluation")
+                nruns_so_far += 1
+                break  # quit
             if exit_info is not None:
                 if num_samples_run > 0:
                     control.model.save_point(x, np.mean(rvec_list[:num_samples_run, :], axis=0), num_samples_run,
@@ -521,7 +555,7 @@ def solve_main(objfun, x0, args, xl, xu, npt, rhobeg, rhoend, maxfun, nruns_so_f
                 break  # quit
 
             # Estimate f in order to compute 'actual reduction'
-            ratio, exit_info = control.calculate_ratio(current_iter, rvec_list[:num_samples_run, :], d, gopt, H)
+            ratio, exit_info = control.calculate_ratio(control.model.xopt(abs_coordinates=True), current_iter, rvec_list[:num_samples_run, :], d, gopt, H)
             if exit_info is not None:
                 if exit_info.able_to_do_restart() and params("restarts.use_restarts") and params(
                         "restarts.use_soft_restarts"):
@@ -545,15 +579,15 @@ def solve_main(objfun, x0, args, xl, xu, npt, rhobeg, rhoend, maxfun, nruns_so_f
 
             # Update delta
             if do_logging:
-                logging.debug("Ratio = %g" % ratio)
+                module_logger.debug("Ratio = %g" % ratio)
             if params("logging.save_diagnostic_info"):
                 diagnostic_info.update_ratio(ratio)
                 diagnostic_info.update_slow_iter(-1)  # n/a, unless otherwise update
             if ratio < params("tr_radius.eta1"):  # ratio < 0.1
                 if finished_growing:
-                    control.delta = min(params("tr_radius.gamma_dec") * control.delta, dnorm)
+                    control.delta = min(params("tr_radius.gamma_dec") * control.delta, dnorm) / tau
                 else:
-                    control.delta = min(params("growing.gamma_dec") * control.delta, dnorm)  # different gamma_dec
+                    control.delta = min(params("growing.gamma_dec") * control.delta, dnorm) / tau  # different gamma_dec
                 if params("logging.save_diagnostic_info"):
                     diagnostic_info.update_iter_type(ITER_ACCEPTABLE_NO_GEOM if ratio > 0.0
                                                      else ITER_UNSUCCESSFUL_NO_GEOM)  # we flag geom update below
@@ -603,7 +637,7 @@ def solve_main(objfun, x0, args, xl, xu, npt, rhobeg, rhoend, maxfun, nruns_so_f
                 knew = control.model.npt()
 
             if do_logging:
-                logging.debug("Updating with knew = %i" % knew)
+                module_logger.debug("Updating with knew = %i" % knew)
             control.model.change_point(knew, xnew, rvec_list[0, :])  # expect step, not absolute x
             for i in range(1, num_samples_run):
                 control.model.add_new_sample(knew, rvec_extra=rvec_list[i, :])
@@ -615,7 +649,7 @@ def solve_main(objfun, x0, args, xl, xu, npt, rhobeg, rhoend, maxfun, nruns_so_f
                     diagnostic_info.update_slow_iter(1 if this_iter_slow else 0)
                 if finished_growing and should_terminate:
                     if do_logging:
-                        logging.info("Slow iteration  - terminating/restarting")
+                        module_logger.info("Slow iteration  - terminating/restarting")
                     if params("restarts.use_restarts") and params("restarts.use_soft_restarts"):
                         number_of_samples = max(nsamples(control.delta, control.rho, current_iter, nruns_so_far), 1)
                         exit_info = control.soft_restart(number_of_samples, nruns_so_far, params,
@@ -637,7 +671,7 @@ def solve_main(objfun, x0, args, xl, xu, npt, rhobeg, rhoend, maxfun, nruns_so_f
                         break  # quit
 
                 # Update list of successful steps
-                this_step_was_not_improvement = control.model.fsave is not None and control.model.fopt() > control.model.fsave
+                this_step_was_not_improvement = control.model.objsave is not None and control.model.objopt() > control.model.objsave
                 succ_steps_not_improvement.pop()  # remove last item
                 succ_steps_not_improvement.insert(0, this_step_was_not_improvement)  # add at beginning
                 # Terminate (not restart) if all are True
@@ -649,7 +683,7 @@ def solve_main(objfun, x0, args, xl, xu, npt, rhobeg, rhoend, maxfun, nruns_so_f
             # While growing, (optionally) add new directions
             if not finished_growing and params("growing.num_new_dirns_each_iter") > 0:
                 if do_logging:
-                    logging.debug("Still growing: adding %g new directions" % params("growing.num_new_dirns_each_iter"))
+                    module_logger.debug("Still growing: adding %g new directions" % params("growing.num_new_dirns_each_iter"))
                 number_of_samples = max(nsamples(control.delta, control.rho, current_iter, nruns_so_far), 1)
                 exit_info = control.add_new_direction_while_growing(number_of_samples, params)
                 if exit_info is not None:
@@ -682,7 +716,7 @@ def solve_main(objfun, x0, args, xl, xu, npt, rhobeg, rhoend, maxfun, nruns_so_f
                                        control.model.npt() - 1)  # cap at number of points
             if finished_growing and ratio > 0.0 and num_regression_steps > 0:
                 if do_logging:
-                    logging.info("Regression: moving %g points" % num_regression_steps)
+                    module_logger.info("Regression: moving %g points" % num_regression_steps)
                 number_of_samples = max(nsamples(control.delta, control.rho, current_iter, nruns_so_far), 1)
                 if params("regression.momentum_extra_steps"):  # move points as random extra steps
                     exit_info = control.move_furthest_points_momentum(d, number_of_samples, num_regression_steps, params)
@@ -729,7 +763,7 @@ def solve_main(objfun, x0, args, xl, xu, npt, rhobeg, rhoend, maxfun, nruns_so_f
                                                                                 np.log(np.maximum(restart_auto_detect_chgJ, 1e-15)))
 
                     if do_logging:
-                        logging.debug("Iter %g: (slope, intercept, r_value) = (%g, %g, %g)" % (current_iter, slope, intercept, r_value))
+                        module_logger.debug("Iter %g: (slope, intercept, r_value) = (%g, %g, %g)" % (current_iter, slope, intercept, r_value))
                     if slope > params("restarts.auto_detect.min_chgJ_slope") \
                             and r_value > params("restarts.auto_detect.min_correl"):
                         # increasing trend, with at least some positive correlation
@@ -740,9 +774,9 @@ def solve_main(objfun, x0, args, xl, xu, npt, rhobeg, rhoend, maxfun, nruns_so_f
                 # Data available (full NumPy vectors of fixed length): restart_auto_detect_delta, restart_auto_detect_chgJ
                 if do_restart and params("restarts.use_soft_restarts"):
                     if do_logging:
-                        logging.info("Auto detection: need to do a restart")
-                        logging.debug("delta history = %s" % str(restart_auto_detect_delta))
-                        logging.debug("chgJ history = %s" % str(restart_auto_detect_chgJ))
+                        module_logger.info("Auto detection: need to do a restart")
+                        module_logger.debug("delta history = %s" % str(restart_auto_detect_delta))
+                        module_logger.debug("chgJ history = %s" % str(restart_auto_detect_chgJ))
                     number_of_samples = max(nsamples(control.delta, control.rho, current_iter, nruns_so_far), 1)
                     exit_info = control.soft_restart(number_of_samples, nruns_so_far, params,
                                                      x_in_abs_coords_to_save=None, rvec_to_save=None,
@@ -759,7 +793,7 @@ def solve_main(objfun, x0, args, xl, xu, npt, rhobeg, rhoend, maxfun, nruns_so_f
                     continue  # next iteration
                 elif do_restart:
                     if do_logging:
-                        logging.info("Auto detection: need to do a restart")
+                        module_logger.info("Auto detection: need to do a restart")
                     exit_info = ExitInformation(EXIT_AUTO_DETECT_RESTART_WARNING, "Auto-detected restart")
                     nruns_so_far += 1
                     break  # quit
@@ -767,7 +801,7 @@ def solve_main(objfun, x0, args, xl, xu, npt, rhobeg, rhoend, maxfun, nruns_so_f
 
             # Otherwise (ratio < eta1 = 0.1), check & fix geometry
             if do_logging:
-                logging.debug("Checking and possibly improving geometry (unsuccessful step)")
+                module_logger.debug("Checking and possibly improving geometry (unsuccessful step)")
             distsq = max((2.0 * control.delta) ** 2, (10.0 * control.rho) ** 2)
             update_delta = False
             number_of_samples = max(nsamples(control.delta, control.rho, current_iter, nruns_so_far), 1)
@@ -812,12 +846,12 @@ def solve_main(objfun, x0, args, xl, xu, npt, rhobeg, rhoend, maxfun, nruns_so_f
                 # Reduce rho
                 control.reduce_rho(current_iter, params)
                 if do_logging:
-                    logging.info("New rho = %g after %i function evaluations" % (control.rho, control.nf))
+                    module_logger.info("New rho = %g after %i function evaluations" % (control.rho, control.nf))
                     if control.n() < params("logging.n_to_print_whole_x_vector"):
-                        logging.debug("Best so far: f = %.15g at x = " % (control.model.fopt())
+                        module_logger.debug("Best so far: f = %.15g at x = " % (control.model.objopt())
                                       + str(control.model.xopt(abs_coordinates=True)))
                     else:
-                        logging.debug("Best so far: f = %.15g at x = [...]" % (control.model.fopt()))
+                        module_logger.debug("Best so far: f = %.15g at x = [...]" % (control.model.objopt()))
                 continue  # next iteration
             else:
                 # Quit on rho=rhoend
@@ -843,16 +877,16 @@ def solve_main(objfun, x0, args, xl, xu, npt, rhobeg, rhoend, maxfun, nruns_so_f
     # (end main loop)
 
     # Quit & return the important information
-    x, rvec, f, jacmin, nsamples = control.model.get_final_results()
+    x, rvec, obj, jacmin, nsamples = control.model.get_final_results()
     if do_logging:
-        logging.debug("At return from DFO-LS, number of function evals = %i" % nf)
-        logging.debug("Smallest objective value = %.15g at x = " % f + str(x))
-    return x, rvec, f, jacmin, nsamples, control.nf, control.nx, nruns_so_far, exit_info, diagnostic_info
+        module_logger.debug("At return from DFO-LS, number of function evals = %i" % nf)
+        module_logger.debug("Smallest objective value = %.15g at x = " % obj + str(x))
+    return x, rvec, obj, jacmin, nsamples, control.nf, control.nx, nruns_so_far, exit_info, diagnostic_info
 
 
-def solve(objfun, x0, args=(), bounds=None, npt=None, rhobeg=None, rhoend=1e-8, maxfun=None, nsamples=None, user_params=None,
+def solve(objfun, x0, h=None, lh=None, prox_uh=None, argsf=(), argsh=(), argsprox=(), bounds=None, projections=[], npt=None, rhobeg=None, rhoend=1e-8, maxfun=None, nsamples=None, user_params=None,
           objfun_has_noise=False, scaling_within_bounds=False, do_logging=True, print_progress=False):
-    x0 = x0.astype(np.float)
+    x0 = x0.astype(float)
     n = len(x0)
 
     # Set missing inputs (if not specified) to some sensible defaults
@@ -861,13 +895,17 @@ def solve(objfun, x0, args=(), bounds=None, npt=None, rhobeg=None, rhoend=1e-8,
         xu = None
     else:
         assert len(bounds) == 2, "bounds must be a 2-tuple of (lower, upper), where both are arrays of size(x0)"
-        xl = bounds[0].astype(np.float) if bounds[0] is not None else None
-        xu = bounds[1].astype(np.float) if bounds[1] is not None else None
+        xl = bounds[0].astype(float) if bounds[0] is not None else None
+        xu = bounds[1].astype(float) if bounds[1] is not None else None
 
     if (xl is None or xu is None) and scaling_within_bounds:
         scaling_within_bounds = False
         warnings.warn("Ignoring scaling_within_bounds=True for unconstrained problem/1-sided bounds", RuntimeWarning)
 
+    if projections and scaling_within_bounds:
+        scaling_within_bounds = False
+        warnings.warn("Ignoring scaling_within_bounds=True for problem with arbitrary constraints", RuntimeWarning)
+
     if xl is None:
         xl = -1e20 * np.ones((n,))  # unconstrained
     if xu is None:
@@ -881,6 +919,18 @@ def solve(objfun, x0, args=(), bounds=None, npt=None, rhobeg=None, rhoend=1e-8,
     if nsamples is None:
         nsamples = lambda delta, rho, iter, nruns: 1  # no averaging
 
+    # If using arbitrary constraints, create projection from bounds
+    if projections:
+        xlb = xl.copy()
+        xub = xu.copy()
+        bproj = lambda w: pbox(w,xlb,xub)
+        projections = list(projections)
+        projections.append(bproj)
+
+        # since using arbitrary constraints, don't constrain otherwise
+        xl = -1e20 * np.ones((n,))
+        xu = 1e20 * np.ones((n,))
+
     # Set parameters
     params = ParameterList(int(n), int(npt), int(maxfun), objfun_has_noise=objfun_has_noise)  # make sure int, not np.int
     if user_params is not None:
@@ -904,13 +954,21 @@ def solve(objfun, x0, args=(), bounds=None, npt=None, rhobeg=None, rhoend=1e-8,
 
     exit_info = None
     # Input & parameter checks
+    if exit_info is None and h is not None:
+        if prox_uh is None:
+            exit_info = ExitInformation(EXIT_INPUT_ERROR, "Must provide prox_uh input if h is not None")
+        elif lh is None:
+            exit_info = ExitInformation(EXIT_INPUT_ERROR, "Must provide lh input if h is not None")
+        elif lh <= 0.0:
+            exit_info = ExitInformation(EXIT_INPUT_ERROR, "lh must be strictly positive")
+
     if exit_info is None and npt < n + 1:
         exit_info = ExitInformation(EXIT_INPUT_ERROR, "npt must be >= n+1 for linear models with inexact interpolation")
 
-    if exit_info is None and rhobeg < 0.0:
+    if exit_info is None and rhobeg <= 0.0:
         exit_info = ExitInformation(EXIT_INPUT_ERROR, "rhobeg must be strictly positive")
 
-    if exit_info is None and rhoend < 0.0:
+    if exit_info is None and rhoend <= 0.0:
         exit_info = ExitInformation(EXIT_INPUT_ERROR, "rhoend must be strictly positive")
 
     if exit_info is None and rhobeg <= rhoend:
@@ -975,13 +1033,20 @@ def solve(objfun, x0, args=(), bounds=None, npt=None, rhobeg=None, rhoend=1e-8,
         results = OptimResults(None, None, None, None, 0, 0, 0, exit_flag, exit_msg)
         return results
 
+    # Enforce arbitrary constraint bounds on x0
+    if projections:
+        xp = dykstra(projections,x0,max_iter=params("dykstra.max_iters"),tol=params("dykstra.d_tol"))
+        if not np.allclose(xp,x0):
+            warnings.warn("x0 not feasible w.r.t given constraints, adjusting", RuntimeWarning)
+            x0 = xp.copy()
+
     # Enforce lower & upper bounds on x0
-    idx = (x0 <= xl)
+    idx = (x0 < xl)
     if np.any(idx):
         warnings.warn("x0 below lower bound, adjusting", RuntimeWarning)
     x0[idx] = xl[idx]
 
-    idx = (x0 >= xu)
+    idx = (x0 > xu)
     if np.any(idx):
         warnings.warn("x0 above upper bound, adjusting", RuntimeWarning)
     x0[idx] = xu[idx]
@@ -991,9 +1056,9 @@ def solve(objfun, x0, args=(), bounds=None, npt=None, rhobeg=None, rhoend=1e-8,
     nruns = 0
     nf = 0
     nx = 0
-    xmin, rmin, fmin, jacmin, nsamples_min, nf, nx, nruns, exit_info, diagnostic_info = \
-        solve_main(objfun, x0, args, xl, xu, npt, rhobeg, rhoend, maxfun, nruns, nf, nx, nsamples, params,
-                    diagnostic_info, scaling_changes, default_growing_method_set_by_user=default_growing_method_set_by_user,
+    xmin, rmin, objmin, jacmin, nsamples_min, nf, nx, nruns, exit_info, diagnostic_info = \
+        solve_main(objfun, x0, argsf, xl, xu, projections, npt, rhobeg, rhoend, maxfun, nruns, nf, nx, nsamples, params,
+                    diagnostic_info, scaling_changes, h, lh, argsh, prox_uh, argsprox, default_growing_method_set_by_user=default_growing_method_set_by_user,
                    do_logging=do_logging, print_progress=print_progress)
 
     # Hard restarts loop
@@ -1007,28 +1072,28 @@ def solve(objfun, x0, args=(), bounds=None, npt=None, rhobeg=None, rhoend=1e-8,
             npt = min(npt, params("restarts.max_npt"))
 
         if do_logging:
-            logging.info("Restarting from finish point (f = %g) after %g function evals; using rhobeg = %g and rhoend = %g"
-                     % (fmin, nf, rhobeg, rhoend))
+            module_logger.info("Restarting from finish point (f = %g) after %g function evals; using rhobeg = %g and rhoend = %g"
+                     % (objmin, nf, rhobeg, rhoend))
         if params("restarts.hard.use_old_rk"):
-            xmin2, rmin2, fmin2, jacmin2, nsamples2, nf, nx, nruns, exit_info, diagnostic_info = \
-                solve_main(objfun, xmin, args, xl, xu, npt, rhobeg, rhoend, maxfun, nruns, nf, nx, nsamples, params,
-                            diagnostic_info, scaling_changes, r0_avg_old=rmin, r0_nsamples_old=nsamples_min,
+            xmin2, rmin2, objmin2, jacmin2, nsamples2, nf, nx, nruns, exit_info, diagnostic_info = \
+                solve_main(objfun, xmin, argsf, xl, xu, projections, npt, rhobeg, rhoend, maxfun, nruns, nf, nx, nsamples, params,
+                            diagnostic_info, scaling_changes, h, lh, argsh, prox_uh, argsprox, r0_avg_old=rmin, r0_nsamples_old=nsamples_min,
                            do_logging=do_logging, print_progress=print_progress)
         else:
-            xmin2, rmin2, fmin2, jacmin2, nsamples2, nf, nx, nruns, exit_info, diagnostic_info = \
-                solve_main(objfun, xmin, args, xl, xu, npt, rhobeg, rhoend, maxfun, nruns, nf, nx, nsamples, params,
-                           diagnostic_info, scaling_changes, do_logging=do_logging, print_progress=print_progress)
+            xmin2, rmin2, objmin2, jacmin2, nsamples2, nf, nx, nruns, exit_info, diagnostic_info = \
+                solve_main(objfun, xmin, argsf, xl, xu, projections, npt, rhobeg, rhoend, maxfun, nruns, nf, nx, nsamples, params,
+                           diagnostic_info, scaling_changes, h, lh, argsh, prox_uh, argsprox, do_logging=do_logging, print_progress=print_progress)
 
-        if fmin2 < fmin or np.isnan(fmin):
+        if objmin2 < objmin or np.isnan(objmin):
             if do_logging:
-                logging.info("Successful run with new f = %s compared to old f = %s" % (fmin2, fmin))
+                module_logger.info("Successful run with new f = %s compared to old f = %s" % (objmin2, objmin))
             last_successful_run = nruns
-            (xmin, rmin, fmin, nsamples_min) = (xmin2, rmin2, fmin2, nsamples2)
+            (xmin, rmin, objmin, nsamples_min) = (xmin2, rmin2, objmin2, nsamples2)
             if jacmin2 is not None:  # may be None if finished during setup phase, in which case just use old Jacobian
                 jacmin = jacmin2
         else:
             if do_logging:
-                logging.info("Unsuccessful run with new f = %s compared to old f = %s" % (fmin2, fmin))
+                module_logger.info("Unsuccessful run with new f = %s compared to old f = %s" % (objmin2, objmin))
 
     if nruns - last_successful_run >= params("restarts.max_unsuccessful_restarts"):
         exit_info = ExitInformation(EXIT_SUCCESS, "Reached maximum number of unsuccessful restarts")
@@ -1040,13 +1105,13 @@ def solve(objfun, x0, args=(), bounds=None, npt=None, rhobeg=None, rhoend=1e-8,
     if scaling_changes is not None and jacmin is not None:
         for i in range(n):
             jacmin[:, i] = jacmin[:, i] / scaling_changes[1][i]
-    results = OptimResults(remove_scaling(xmin, scaling_changes), rmin, fmin, jacmin, nf, nx, nruns, exit_flag, exit_msg)
+    results = OptimResults(remove_scaling(xmin, scaling_changes), rmin, objmin, jacmin, nf, nx, nruns, exit_flag, exit_msg)
     if params("logging.save_diagnostic_info"):
         df = diagnostic_info.to_dataframe(with_xk=params("logging.save_xk"), with_rk=params("logging.save_rk"))
         results.diagnostic_info = df
 
     if do_logging:
-        logging.info("Did a total of %g run(s)" % nruns)
+        module_logger.info("Did a total of %g run(s)" % nruns)
 
     return results