DFO-LS 1.5.3__tar.gz → 1.6__tar.gz

{dfo_ls-1.5.3 → dfo_ls-1.6}/DFO_LS.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.4
 Name: DFO-LS
-Version: 1.5.3
+Version: 1.6
 Summary: A flexible derivative-free solver for (bound constrained) nonlinear least-squares minimization
 Author-email: Lindon Roberts <lindon.roberts@sydney.edu.au>
 Maintainer-email: Lindon Roberts <lindon.roberts@sydney.edu.au>
@@ -41,6 +41,7 @@ Requires-Dist: Sphinx; extra == "dev"
 Requires-Dist: sphinx-rtd-theme; extra == "dev"
 Provides-Extra: trustregion
 Requires-Dist: trustregion>=1.1; extra == "trustregion"
+Dynamic: license-file
 
 ===================================================
 DFO-LS: Derivative-Free Optimizer for Least-Squares

{dfo_ls-1.5.3 → dfo_ls-1.6}/DFO_LS.egg-info/SOURCES.txt

@@ -10,6 +10,7 @@ DFO_LS.egg-info/top_level.txt
 dfols/__init__.py
 dfols/controller.py
 dfols/diagnostic_info.py
+dfols/evaluations_database.py
 dfols/hessian.py
 dfols/model.py
 dfols/params.py

{dfo_ls-1.5.3 → dfo_ls-1.6}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.4
 Name: DFO-LS
-Version: 1.5.3
+Version: 1.6
 Summary: A flexible derivative-free solver for (bound constrained) nonlinear least-squares minimization
 Author-email: Lindon Roberts <lindon.roberts@sydney.edu.au>
 Maintainer-email: Lindon Roberts <lindon.roberts@sydney.edu.au>
@@ -41,6 +41,7 @@ Requires-Dist: Sphinx; extra == "dev"
 Requires-Dist: sphinx-rtd-theme; extra == "dev"
 Provides-Extra: trustregion
 Requires-Dist: trustregion>=1.1; extra == "trustregion"
+Dynamic: license-file
 
 ===================================================
 DFO-LS: Derivative-Free Optimizer for Least-Squares

{dfo_ls-1.5.3 → dfo_ls-1.6}/dfols/__init__.py

@@ -39,9 +39,11 @@ alternative licensing.
 from __future__ import absolute_import, division, print_function, unicode_literals
 
 # DFO-LS version
-__version__ = '1.5.3'
+__version__ = '1.6'
 
 # Main solver & exit flags
 from .solver import *
-__all__ = ['solve']
+__all__ = ['solve', 'OptimResults']
 
+from .evaluations_database import *
+__all__ += ['EvaluationDatabase']

{dfo_ls-1.5.3 → dfo_ls-1.6}/dfols/controller.py

@@ -414,6 +414,48 @@ class Controller(object):
 
         return None
 
+    def initialise_from_database(self, eval_database, number_of_samples, params):
+        # Here, eval_database has at least one entry, and the base index has already been used
+        # to evaluate (x0,r0), which has already been added to self.model
+        # Now, find exactly n feasible perturbations (either from database or new evals) and add them to the model
+        base_idx, perturbation_idx, new_perturbations = eval_database.select_starting_evals(self.delta,
+                                                                                            xl=self.model.xbase + self.model.sl,
+                                                                                            xu=self.model.xbase + self.model.su,
+                                                                                            projections=self.model.projections,
+                                                                                            tol=params("database.new_direction_tol"),
+                                                                                            dykstra_max_iters=params("dykstra.max_iters"),
+                                                                                            dykstra_tol=params("dykstra.d_tol"))
+
+        # Add suitable pre-existing evaluations
+        for i, idx in enumerate(perturbation_idx):
+            module_logger.info("Adding pre-existing evaluation %g to initial model" % idx)
+            x, rx = eval_database.get_eval(idx)
+            self.model.change_point(i + 1, x - self.model.xbase, rx, -idx)  # use eval_num = -idx
+
+        if new_perturbations is not None:
+            num_perturbations = new_perturbations.shape[0]
+            module_logger.debug("Adding %g new evaluations to initial model" % num_perturbations)
+            for i in range(num_perturbations):
+                new_point = (eval_database.get_x(base_idx) - self.model.xbase) + new_perturbations[i,:]  # new_perturbations[i,:] has length <= self.delta
+
+                # Evaluate objective
+                x = self.model.as_absolute_coordinates(new_point)
+                rvec_list, obj_list, num_samples_run, exit_info = self.evaluate_objective(x, number_of_samples, params)
+
+                # Handle exit conditions (f < min obj value or maxfun reached)
+                if exit_info is not None:
+                    if num_samples_run > 0:
+                        self.model.save_point(x, np.mean(rvec_list[:num_samples_run, :], axis=0), num_samples_run,
+                                              self.nx, x_in_abs_coords=True)
+                    return exit_info  # return & quit
+
+                # Otherwise, add new results (increments model.npt_so_far)
+                self.model.change_point(len(perturbation_idx) + 1 + i, x - self.model.xbase, rvec_list[0, :], self.nx)  # expect step, not absolute x
+                for j in range(1, num_samples_run):
+                    self.model.add_new_sample(len(perturbation_idx) + 1 + i, rvec_extra=rvec_list[j, :])
+
+        return None
+
     def add_new_direction_while_growing(self, number_of_samples, params, min_num_steps=0):
         num_steps = max(params('growing.num_new_dirns_each_iter'), min_num_steps)
         step_length = params('growing.delta_scale_new_dirns') * self.delta

dfo_ls-1.6/dfols/evaluations_database.py

@@ -0,0 +1,208 @@
+"""
+Class to create/store database of existing evaluations, and routines to select
+existing evaluations to build an initial linear model
+"""
+import logging
+import numpy as np
+
+from .util import apply_scaling, dykstra
+from .trust_region import ctrsbox_geometry, trsbox_geometry
+
+__all__ = ['EvaluationDatabase']
+
+module_logger = logging.getLogger(__name__)
+
+
+# Class to store set of evaluations (x, rx)
+class EvaluationDatabase(object):
+    def __init__(self, eval_list=None, starting_eval=None):
+        # eval_list is a list of tuples (x, rx)
+        self._evals = []
+        if eval_list is not None:
+            for e in eval_list:
+                self._evals.append(e)
+
+        # Which evaluation index should be the starting point of the optimization?
+        self.starting_eval = None
+        if starting_eval is not None and 0 <= starting_eval <= len(self._evals):
+            self.starting_eval = starting_eval
+
+    def __len__(self):
+        return len(self._evals)
+
+    def append(self, x, rx, make_starting_eval=False):
+        self._evals.append((x, rx))
+        if make_starting_eval:
+            self.starting_eval = len(self) - 1
+
+    def set_starting_eval(self, index):
+        if 0 <= index < len(self):
+            self.starting_eval = index
+        else:
+            raise IndexError("Invalid index %g given current set of %g evaluations" % (index, len(self)))
+
+    def get_starting_eval_idx(self):
+        if len(self) == 0:
+            raise RuntimeError("No evaluations available, no suitable starting evaluation ")
+        elif self.starting_eval is None:
+            module_logger.warning("Starting evaluation index not set, using most recently appended evaluation")
+            self.starting_eval = len(self) - 1
+
+        return self.starting_eval
+
+    def get_eval(self, index):
+        # Return (x, rx) for given index
+        if 0 <= index < len(self):
+            return self._evals[index][0], self._evals[index][1]
+        else:
+            raise IndexError("Invalid index %g given current set of %g evaluations" % (index, len(self)))
+
+    def get_x(self, index):
+        return self.get_eval(index)[0]
+
+    def get_rx(self, index):
+        return self.get_eval(index)[1]
+
+    def apply_scaling(self, scaling_changes):
+        # Adjust all input x values based on scaling
+        if scaling_changes is not None:
+            for i in range(len(self)):
+                x, rx = self._evals[i]
+                self._evals[i] = (apply_scaling(x, scaling_changes), rx)
+        return
+
+    def select_starting_evals(self, delta, xl=None, xu=None, projections=[], tol=1e-8,
+                              dykstra_max_iters=100, dykstra_tol=1e-10):
+        # Given a database 'evals' with prescribed starting index, and initial trust-region radius delta > 0
+        # determine a subset of the database to use
+
+        # The bounds xl <= x <= xu and projection list are used to determine where to evaluate any new points
+        # (ensuring they are feasible)
+
+        if delta <= 0.0:
+            raise RuntimeError("delta must be strictly positive")
+        if len(self) == 0:
+            raise RuntimeError("Need at least one evaluation to select starting evaluations")
+
+        base_idx = self.get_starting_eval_idx()
+        xbase = self.get_x(self.get_starting_eval_idx())
+        n = len(xbase)
+        module_logger.debug("Selecting starting evaluations from existing database")
+        module_logger.debug("Have %g evaluations to choose from" % len(self))
+        module_logger.debug("Using base index %g" % base_idx)
+
+        # For linear interpolation, we will use the matrix
+        # M = [[1, 0], [0, L]] where L has rows (xi-xbase)/delta
+        # So, just build a large matrix Lfull with everything
+        n_perturbations = len(self) - 1
+        Lfull = np.zeros((n_perturbations, n))
+        row_idx = 0
+        for i in range(n_perturbations + 1):
+            if i == base_idx:
+                continue
+            Lfull[row_idx, :] = (self.get_x(i) - xbase) / delta  # Lfull[i,:] = (xi-xbase) / delta
+            row_idx += 1
+
+        xdist = np.linalg.norm(Lfull, axis=1)  # xdist[i] = ||Lfull[i,:]|| = ||xi-xbase|| / delta
+        # module_logger.debug("xdist =", xdist)
+
+        # We ideally want xdist ~ 1, so reweight these distances based on that (large xdist_reweighted --> xdist ~ 1 --> good)
+        xdist_reweighted = 1.0 / np.maximum(xdist, 1.0 / xdist)
+        # module_logger.debug("xdist_reweighted =", xdist_reweighted)
+
+        if n_perturbations == 0:
+            module_logger.debug("Only one evaluation available, just selecting that")
+            return base_idx, [], delta * np.eye(n)
+
+        # Now, find as many good perturbations as we can
+        # Good = not too far from xbase (relative to delta) and sufficiently linearly independent
+        #        from other selected perturbations (i.e. Lfull[perturbation_idx,:] well-conditioned
+        #        and len(perturbation_idx) <= n
+        perturbation_idx = []  # what point indices to use as perturbations
+
+        for iter in range(min(n_perturbations, n)):
+            # Add one more good perturbation, if available
+            # Note: can only add at most the number of available perturbations, or n perturbations, whichever is smaller
+            if iter == 0:
+                # First perturbation: every direction is equally good, so pick the point closest to the
+                # trust-region boundary
+                idx = int(np.argmax(xdist_reweighted))
+                module_logger.debug("Adding index %g with ||xi-xbase|| / delta = %g" % (idx if idx < base_idx else idx+1, xdist[idx]))
+                perturbation_idx.append(idx)
+            else:
+                Q, R = np.linalg.qr(Lfull[perturbation_idx, :].T, mode='reduced')
+                # module_logger.debug("Current perturbation_idx =", perturbation_idx)
+                L_rem = Lfull @ (np.eye(n) - Q @ Q.T)  # part of (xi-xbase)/delta orthogonal to current perturbations
+                # rem_size = fraction of original length ||xi-xbase||/delta that is orthogonal to current perturbations
+                # all entries are in [0,1], and is zero for already selected perturbations
+                rem_size = np.linalg.norm(L_rem, axis=1) / xdist
+                rem_size[perturbation_idx] = 0  # ensure this holds exactly
+                # module_logger.debug("rem_size =", rem_size)
+                # module_logger.debug("rem_size * xdist_reweighted =", rem_size * xdist_reweighted)
+
+                # We want a point with large rem_size and xdist ~ 1 (i.e. xdist_reweighted large)
+                idx = int(np.argmax(rem_size * xdist_reweighted))
+                if rem_size[idx] * xdist_reweighted[idx] > tol:
+                    # This ensures new perturbation is sufficiently linearly independent of existing perturbations
+                    # (and also ensures idx hasn't already been chosen)
+                    module_logger.debug("Adding index %g" % (idx if idx < base_idx else idx+1))
+                    perturbation_idx.append(idx)
+                else:
+                    module_logger.debug("No more linearly independent directions, quitting")
+                    break
+
+        # Find new linearly independent directions
+        if len(perturbation_idx) < n:
+            module_logger.debug("Selecting %g new linearly independent directions" % (n - len(perturbation_idx)))
+            Q, _ = np.linalg.qr(Lfull[perturbation_idx, :].T, mode='complete')
+            new_perturbations = delta * Q[:, len(perturbation_idx):].T
+
+            # Make perturbations feasible w.r.t. xl <= x <= xu and projections
+            # Note: if len(projections) > 0, then the projection list *already* includes bounds
+            # Don't need to make pre-existing evaluations feasible, since we already have r(x) for these
+
+            # Start construction of interpolation matrix for later
+            L = np.zeros((n, n), dtype=float)
+            L[:len(perturbation_idx), :] = Lfull[perturbation_idx, :]
+            L[len(perturbation_idx):, :] = new_perturbations / delta
+
+            # Since we already have a full set of linearly independent directions,
+            # we do this by moving each infeasible perturbation to a geometry-improving location
+            for i in range(new_perturbations.shape[0]):
+                xnew = xbase + new_perturbations[i, :]
+                # Check feasibility
+                if len(projections) == 0:
+                    # Bounds only
+                    feasible = np.all(xnew >= xl) and np.all(xnew <= xu)
+                else:
+                    # Projections
+                    xnew_C = dykstra(projections, xnew, max_iter=dykstra_max_iters, tol=dykstra_tol)
+                    feasible = np.linalg.norm(xnew - xnew_C) < dykstra_tol
+
+                if feasible:
+                    # Skip feasible points, nothing to do
+                    continue
+
+                # If infeasible, build Lagrange polynomial and move to geometry-improving location in B(xbase,delta)
+                # which will automatically be feasible
+                module_logger.debug("Moving default %g-th new perturbation to ensure feasibility" % i)
+                c = 0.0  # Lagrange polynomial centered at xbase
+                ei = np.zeros((n,), dtype=float)
+                ei[len(perturbation_idx) + i] = 1.0
+                g = np.linalg.solve(L, ei) / delta  # divide by delta because L is scaled by 1/delta
+                if len(projections) == 0:
+                    new_perturbations[i, :] = trsbox_geometry(xbase, c, g, xl, xu, delta)
+                else:
+                    new_perturbations[i, :] = ctrsbox_geometry(xbase, c, g, projections, delta)
+
+                # Update L after replacement
+                L[len(perturbation_idx) + i, :] = new_perturbations[i,:] / delta
+        else:
+            module_logger.debug("Full set of directions found, no need for new evaluations")
+            new_perturbations = None
+
+        # perturbation_idx in [0, ..., n_perturbations-1], reset to be actual indices
+        for i in range(len(perturbation_idx)):
+            if perturbation_idx[i] >= base_idx:
+                perturbation_idx[i] += 1
+        return base_idx, perturbation_idx, new_perturbations

{dfo_ls-1.5.3 → dfo_ls-1.6}/dfols/params.py

@@ -122,6 +122,9 @@ class ParameterList(object):
         self.params["func_tol.tr_step"] = 1-1e-1
         self.params["func_tol.max_iters"] = 500
         self.params["sfista.max_iters_scaling"] = 2.0
+
+        # Evaluation database
+        self.params["database.new_direction_tol"] = 1e-8
         
         self.params_changed = {}
         for p in self.params:
@@ -284,6 +287,8 @@ class ParameterList(object):
             type_str, nonetype_ok, lower, upper = 'int', False, 0, None
         elif key == "sfista.max_iters_scaling":
             type_str, nonetype_ok, lower, upper = 'float', False, 1.0, None
+        elif key == "database.new_direction_tol":
+            type_str, nonetype_ok, lower, upper = 'float', False, 0.0, None
         else:
             assert False, "ParameterList.param_type() has unknown key: %s" % key
         return type_str, nonetype_ok, lower, upper

{dfo_ls-1.5.3 → dfo_ls-1.6}/dfols/solver.py

@@ -32,16 +32,18 @@ import logging
 from math import sqrt
 import numpy as np
 import os
+import pandas as pd
 import scipy.linalg as LA
 import scipy.stats as STAT
 import warnings
 
 from .controller import *
 from .diagnostic_info import *
+from .evaluations_database import *
 from .params import *
 from .util import *
 
-__all__ = ['solve']
+__all__ = ['solve', 'OptimResults']
 
 module_logger = logging.getLogger(__name__) 
 
@@ -69,13 +71,16 @@ class OptimResults(object):
         self.EXIT_TR_INCREASE_ERROR = EXIT_TR_INCREASE_ERROR
         self.EXIT_LINALG_ERROR = EXIT_LINALG_ERROR
         self.EXIT_FALSE_SUCCESS_WARNING = EXIT_FALSE_SUCCESS_WARNING
+        self.max_resid_length_print = 20  # don't print self.resid in __str__ if length >= this value
+        self.max_jac_length_print = 40  # don't print self.jacobian in __str__ if length >= this value
+
 
     def __str__(self):
         # Result of calling print(soln)
         output = "****** DFO-LS Results ******\n"
         if self.flag != self.EXIT_INPUT_ERROR:
             output += "Solution xmin = %s\n" % str(self.x)
-            if len(self.resid) < 100:
+            if len(self.resid) < self.max_resid_length_print:
                 output += "Residual vector = %s\n" % str(self.resid)
             else:
                 output += "Not showing residual vector because it is too long; check self.resid\n"
@@ -83,7 +88,7 @@ class OptimResults(object):
             output += "Needed %g objective evaluations (at %g points)\n" % (self.nf, self.nx)
             if self.nruns > 1:
                 output += "Did a total of %g runs\n" % self.nruns
-            if self.jacobian is not None and np.size(self.jacobian) < 200:
+            if self.jacobian is not None and np.size(self.jacobian) < self.max_jac_length_print:
                 output += "Approximate Jacobian = %s\n" % str(self.jacobian)
             elif self.jacobian is None:
                 output += "No Jacobian returned\n"
@@ -92,69 +97,136 @@ class OptimResults(object):
             if self.diagnostic_info is not None:
                 output += "Diagnostic information available; check self.diagnostic_info\n"
             output += "Solution xmin was evaluation point %g\n" % self.xmin_eval_num
-            if len(self.jacmin_eval_nums) < 100:
+            if self.jacmin_eval_nums is not None and len(self.jacmin_eval_nums) < self.max_resid_length_print:
                 output += "Approximate Jacobian formed using evaluation points %s\n" % str(self.jacmin_eval_nums)
+            elif self.jacmin_eval_nums is None:
+                output += "Approximate Jacobian not formed using problem information, disregard\n"
+            else:
+                output += "Not showing Jacobian evaluation points because it is too long; check self.jacmin_eval_nums\n"
         output += "Exit flag = %g\n" % self.flag
         output += "%s\n" % self.msg
         output += "****************************\n"
         return output
+    
+    def to_dict(self, replace_nan=True):
+        # Convert to a serializable dict object suitable for saving in a json file
+        # If replace_nan=True, convert all NaN entries to None
+        soln_dict = {}
+        soln_dict['x'] = self.x.tolist() if self.x is not None else None
+        soln_dict['resid'] = self.resid.tolist() if self.resid is not None else None
+        soln_dict['obj'] = float(self.obj)
+        soln_dict['jacobian'] = self.jacobian.tolist() if self.jacobian is not None else None
+        soln_dict['nf'] = int(self.nf)
+        soln_dict['nx'] = int(self.nx)
+        soln_dict['nruns'] = int(self.nruns)
+        soln_dict['flag'] = int(self.flag)
+        soln_dict['msg'] = str(self.msg)
+        soln_dict['diagnostic_info'] = self.diagnostic_info.to_dict() if self.diagnostic_info is not None else None
+        soln_dict['xmin_eval_num'] = int(self.xmin_eval_num)
+        soln_dict['jacmin_eval_nums'] = self.jacmin_eval_nums.tolist() if self.jacmin_eval_nums is not None else None
+        if replace_nan:
+            return replace_nan_with_none(soln_dict)
+        else:
+            return soln_dict
+    
+    @staticmethod
+    def from_dict(soln_dict):
+        # Take a dict object containing OptimResults information, and return the relevant OptimResults object
+        # Input soln_dict should come from soln.to_dict()
+        # Note: np.array(mylist, dtype=float) automatically converts None to NaN
+        x = np.array(soln_dict['x'], dtype=float) if soln_dict['x'] is not None else None
+        resid = np.array(soln_dict['resid'], dtype=float) if soln_dict['resid'] is not None else None
+        obj = soln_dict['obj']
+        jacobian = np.array(soln_dict['jacobian'], dtype=float) if soln_dict['jacobian'] is not None else None
+        nf = soln_dict['nf']
+        nx = soln_dict['nx']
+        nruns = soln_dict['nruns']
+        flag = soln_dict['flag']
+        msg = soln_dict['msg']
+        xmin_eval_num = soln_dict['xmin_eval_num']
+        jacmin_eval_nums = np.array(soln_dict['jacmin_eval_nums'], dtype=int) if soln_dict['jacmin_eval_nums'] is not None else None
+        
+        soln = OptimResults(x, resid, obj, jacobian, nf, nx, nruns, flag, msg, xmin_eval_num, jacmin_eval_nums)
+        
+        if soln_dict['diagnostic_info'] is not None:
+            soln.diagnostic_info = pd.DataFrame.from_dict(soln_dict['diagnostic_info'])
+        return soln
 
 
 def solve_main(objfun, x0, argsf, xl, xu, projections, npt, rhobeg, rhoend, maxfun, nruns_so_far, nf_so_far, nx_so_far, nsamples, params,
                diagnostic_info, scaling_changes, h=None, lh=None, argsh=(), prox_uh=None, argsprox=None, r0_avg_old=None, r0_nsamples_old=None, default_growing_method_set_by_user=None,
                do_logging=True, print_progress=False):
+
+    if type(x0) == EvaluationDatabase:
+        x0_is_eval_database = True
+        x0_vec = x0.get_x(x0.get_starting_eval_idx())
+    else:
+        x0_vec = x0
+        x0_is_eval_database = False
+    n = len(x0_vec)
+
     # Evaluate at x0 (keep nf, nx correct and check for f < 1e-12)
     # The hard bit is determining what m = len(r0) should be, and allocating memory appropriately
     if r0_avg_old is None:
-        number_of_samples = max(nsamples(rhobeg, rhobeg, 0, nruns_so_far), 1)
-        # Evaluate the first time...
-        nf = nf_so_far + 1
-        nx = nx_so_far + 1
-        r0, obj0 = eval_least_squares_with_regularisation(objfun, remove_scaling(x0, scaling_changes), h, 
-                                              argsf=argsf, argsh=argsh, verbose=do_logging, eval_num=nf, pt_num=nx,
-                                              full_x_thresh=params("logging.n_to_print_whole_x_vector"),
-                                              check_for_overflow=params("general.check_objfun_for_overflow"))
-        m = len(r0)
-
-        # Now we have m, we can evaluate the rest of the times
-        rvec_list = np.zeros((number_of_samples, m))
-        obj_list = np.zeros((number_of_samples,))
-        rvec_list[0, :] = r0
-        obj_list[0] = obj0
-        num_samples_run = 1
         exit_info = None
+        if x0_is_eval_database:
+            # We have already got r(x0), so just extract this information
+            nf = nf_so_far
+            nx = nx_so_far
+            num_samples_run = 1
+            r0_avg = x0.get_rx(x0.get_starting_eval_idx())
+            m = len(r0_avg)
+            module_logger.info("Using pre-existing evaluation %g as starting point" % (x0.get_starting_eval_idx()))
+        else:
+            number_of_samples = max(nsamples(rhobeg, rhobeg, 0, nruns_so_far), 1)
+            # Evaluate the first time...
+            nf = nf_so_far + 1
+            nx = nx_so_far + 1
+            r0, obj0 = eval_least_squares_with_regularisation(objfun, remove_scaling(x0_vec, scaling_changes), h,
+                                                  argsf=argsf, argsh=argsh, verbose=do_logging, eval_num=nf, pt_num=nx,
+                                                  full_x_thresh=params("logging.n_to_print_whole_x_vector"),
+                                                  check_for_overflow=params("general.check_objfun_for_overflow"))
+            m = len(r0)
+
+            # Now we have m, we can evaluate the rest of the times
+            rvec_list = np.zeros((number_of_samples, m))
+            obj_list = np.zeros((number_of_samples,))
+            rvec_list[0, :] = r0
+            obj_list[0] = obj0
+            num_samples_run = 1
+
+            for i in range(1, number_of_samples):  # skip first eval - already did this
+                if nf >= maxfun:
+                    exit_info = ExitInformation(EXIT_MAXFUN_WARNING, "Objective has been called MAXFUN times")
+                    nruns_so_far += 1
+                    break  # stop evaluating at x0
 
-        for i in range(1, number_of_samples):  # skip first eval - already did this
-            if nf >= maxfun:
-                exit_info = ExitInformation(EXIT_MAXFUN_WARNING, "Objective has been called MAXFUN times")
-                nruns_so_far += 1
-                break  # stop evaluating at x0
+                nf += 1
+                # Don't increment nx for x0 - we did this earlier
+                rvec_list[i, :], obj_list[i] = eval_least_squares_with_regularisation(objfun, remove_scaling(x0_vec, scaling_changes), h,
+                                                    argsf=argsf, argsh=argsh, verbose=do_logging, eval_num=nf, pt_num=nx,
+                                                    full_x_thresh=params("logging.n_to_print_whole_x_vector"),
+                                                    check_for_overflow=params("general.check_objfun_for_overflow"))
+                num_samples_run += 1
 
-            nf += 1
-            # Don't increment nx for x0 - we did this earlier
-            rvec_list[i, :], obj_list[i] = eval_least_squares_with_regularisation(objfun, remove_scaling(x0, scaling_changes), h, 
-                                                argsf=argsf, argsh=argsh, verbose=do_logging, eval_num=nf, pt_num=nx,
-                                                full_x_thresh=params("logging.n_to_print_whole_x_vector"),
-                                                check_for_overflow=params("general.check_objfun_for_overflow"))
-            num_samples_run += 1
+            r0_avg = np.mean(rvec_list[:num_samples_run, :], axis=0)
 
-        r0_avg = np.mean(rvec_list[:num_samples_run, :], axis=0)
         # NOTE: modify objvalue here
         if h is None:
             if sumsq(r0_avg) <= params("model.abs_tol"):
                 exit_info = ExitInformation(EXIT_SUCCESS, "Objective is sufficiently small")
         else:
-            if sumsq(r0_avg) + h(remove_scaling(x0, scaling_changes), *argsh)<= params("model.abs_tol"):
+            if sumsq(r0_avg) + h(remove_scaling(x0_vec, scaling_changes), *argsh)<= params("model.abs_tol"):
                 exit_info = ExitInformation(EXIT_SUCCESS, "Objective is sufficiently small")
 
         if exit_info is not None:
             xmin_eval_num = 0
             jacmin_eval_nums = np.array([0], dtype=int)
-            return x0, r0_avg, sumsq(r0_avg), None, num_samples_run, nf, nx, nruns_so_far+1, exit_info, diagnostic_info, xmin_eval_num, jacmin_eval_nums
+            return x0_vec, r0_avg, sumsq(r0_avg), None, num_samples_run, nf, nx, nruns_so_far+1, exit_info, diagnostic_info, xmin_eval_num, jacmin_eval_nums
 
     else:  # have old r0 information (e.g. from previous restart), use this instead
 
-        # m = len(r0_avg_old)
+        m = len(r0_avg_old)
         r0_avg = r0_avg_old
         num_samples_run = r0_nsamples_old
         nf = nf_so_far
@@ -164,7 +236,7 @@ def solve_main(objfun, x0, argsf, xl, xu, projections, npt, rhobeg, rhoend, maxf
     if default_growing_method_set_by_user is not None and (not default_growing_method_set_by_user):
         # If m>=n, the default growing method (use_full_rank_interp) is best
         # However, this can fail for m<n, so need to use an alternative method (perturb_trust_region_step)
-        if m < len(x0):
+        if m < n:
             if do_logging:
                 module_logger.debug("Inverse problem (m<n), switching default growing method")
             params('growing.full_rank.use_full_rank_interp', new_value=False)
@@ -173,25 +245,32 @@ def solve_main(objfun, x0, argsf, xl, xu, projections, npt, rhobeg, rhoend, maxf
                 params('growing.delta_scale_new_dirns', new_value=0.1)
 
     # Initialise controller
-    control = Controller(objfun, argsf, x0, r0_avg, num_samples_run, xl, xu, projections, npt, rhobeg, rhoend, nf, nx, maxfun,
+    control = Controller(objfun, argsf, x0_vec, r0_avg, num_samples_run, xl, xu, projections, npt, rhobeg, rhoend, nf, nx, maxfun,
                          params, scaling_changes, do_logging, h=h, lh=lh, argsh=argsh,  prox_uh=prox_uh, argsprox=argsprox)
 
     # Initialise interpolation set
     number_of_samples = max(nsamples(control.delta, control.rho, 0, nruns_so_far), 1)
     num_directions = min(params("growing.ndirs_initial") + params("restarts.hard.increase_ndirs_initial_amt") * nruns_so_far,
                          npt - 1)  # cap at npt
-    if params("init.random_initial_directions"):
-        if do_logging:
-            module_logger.info("Initialising (random directions)")
-        exit_info = control.initialise_random_directions(number_of_samples, num_directions, params)
+    if x0_is_eval_database:
+        if num_directions != n:
+            module_logger.warning("When evaluation database provided, we will always initialize with n+1 evaluations")
+        exit_info = control.initialise_from_database(x0, number_of_samples, params)
     else:
-        if do_logging:
-            module_logger.info("Initialising (coordinate directions)")
-        exit_info = control.initialise_coordinate_directions(number_of_samples, num_directions, params)
+        if params("init.random_initial_directions"):
+            if do_logging:
+                module_logger.info("Initialising (random directions)")
+            exit_info = control.initialise_random_directions(number_of_samples, num_directions, params)
+        else:
+            if do_logging:
+                module_logger.info("Initialising (coordinate directions)")
+            exit_info = control.initialise_coordinate_directions(number_of_samples, num_directions, params)
     if exit_info is not None:
         x, rvec, obj, jacmin, nsamples, x_eval_num, jac_eval_nums = control.model.get_final_results()
         return x, rvec, obj, None, nsamples, control.nf, control.nx, nruns_so_far + 1, exit_info, diagnostic_info, x_eval_num, jac_eval_nums
 
+    # model.npt() = actual number of evaluations available to the model so far
+    # model.num_pts = desired interp set size >= n+1
     finished_growing = (control.model.npt() >= control.model.num_pts)  # have we finished growing the initial set yet?
 
     # Save list of last N successful steps: whether they failed to be an improvement over fsave
@@ -893,8 +972,16 @@ def solve_main(objfun, x0, argsf, xl, xu, projections, npt, rhobeg, rhoend, maxf
 
 def solve(objfun, x0, h=None, lh=None, prox_uh=None, argsf=(), argsh=(), argsprox=(), bounds=None, projections=[], npt=None, rhobeg=None, rhoend=1e-8, maxfun=None, nsamples=None, user_params=None,
           objfun_has_noise=False, scaling_within_bounds=False, do_logging=True, print_progress=False):
-    x0 = x0.astype(float)
-    n = len(x0)
+
+    if type(x0) == EvaluationDatabase:
+        assert len(x0) > 0, "evaluation database x0 cannot be empty"
+        assert 0 <= x0.get_starting_eval_idx() < len(x0), "evaluation database must have valid starting index set"
+        x0_is_eval_database = True
+        n = len(x0.get_x(x0.get_starting_eval_idx()))
+    else:
+        x0 = np.array(x0).astype(float)
+        n = len(x0)
+        x0_is_eval_database = False
 
     # Set missing inputs (if not specified) to some sensible defaults
     if bounds is None:
@@ -920,7 +1007,8 @@ def solve(objfun, x0, h=None, lh=None, prox_uh=None, argsf=(), argsh=(), argspro
     if npt is None:
         npt = n + 1
     if rhobeg is None:
-        rhobeg = 0.1 if scaling_within_bounds else 0.1 * max(np.max(np.abs(x0)), 1.0)
+        x0_norm = np.max(np.abs(x0.get_x(x0.get_starting_eval_idx()))) if x0_is_eval_database else np.max(np.abs(x0))
+        rhobeg = 0.1 if scaling_within_bounds else 0.1 * max(x0_norm, 1.0)
     if maxfun is None:
         maxfun = min(100 * (n + 1), 1000)  # 100 gradients, capped at 1000
     if nsamples is None:
@@ -955,7 +1043,10 @@ def solve(objfun, x0, h=None, lh=None, prox_uh=None, argsf=(), argsh=(), argspro
         scale = xu - xl
         scaling_changes = (shift, scale)
 
-    x0 = apply_scaling(x0, scaling_changes)
+    if x0_is_eval_database:
+        x0.apply_scaling(scaling_changes)
+    else:
+        x0 = apply_scaling(x0, scaling_changes)
     xl = apply_scaling(xl, scaling_changes)
     xu = apply_scaling(xu, scaling_changes)
 
@@ -984,13 +1075,19 @@ def solve(objfun, x0, h=None, lh=None, prox_uh=None, argsf=(), argsh=(), argspro
     if exit_info is None and maxfun <= 0:
         exit_info = ExitInformation(EXIT_INPUT_ERROR, "maxfun must be strictly positive")
 
-    if exit_info is None and np.shape(x0) != (n,):
-        exit_info = ExitInformation(EXIT_INPUT_ERROR, "x0 must be a vector")
+    if exit_info is None:
+        if x0_is_eval_database:
+            for i in range(len(x0)):
+                if np.shape(x0.get_x(i)) != (n,):
+                    exit_info = ExitInformation(EXIT_INPUT_ERROR, "All input vectors x0 must have the same shape")
+        else:
+            if np.shape(x0) != (n,):
+                exit_info = ExitInformation(EXIT_INPUT_ERROR, "x0 must be a vector")
 
-    if exit_info is None and np.shape(x0) != np.shape(xl):
+    if exit_info is None and np.shape(xl) != (n,):
         exit_info = ExitInformation(EXIT_INPUT_ERROR, "lower bounds must have same shape as x0")
 
-    if exit_info is None and np.shape(x0) != np.shape(xu):
+    if exit_info is None and np.shape(xu) != (n,):
         exit_info = ExitInformation(EXIT_INPUT_ERROR, "upper bounds must have same shape as x0")
 
     if exit_info is None and np.min(xu - xl) < 2.0 * rhobeg:
@@ -1041,22 +1138,24 @@ def solve(objfun, x0, h=None, lh=None, prox_uh=None, argsf=(), argsh=(), argspro
         return results
 
     # Enforce arbitrary constraint bounds on x0
-    if projections:
-        xp = dykstra(projections,x0,max_iter=params("dykstra.max_iters"),tol=params("dykstra.d_tol"))
-        if not np.allclose(xp,x0):
-            warnings.warn("x0 not feasible w.r.t given constraints, adjusting", RuntimeWarning)
-            x0 = xp.copy()
-
-    # Enforce lower & upper bounds on x0
-    idx = (x0 < xl)
-    if np.any(idx):
-        warnings.warn("x0 below lower bound, adjusting", RuntimeWarning)
-    x0[idx] = xl[idx]
-
-    idx = (x0 > xu)
-    if np.any(idx):
-        warnings.warn("x0 above upper bound, adjusting", RuntimeWarning)
-    x0[idx] = xu[idx]
+    if not x0_is_eval_database:
+        # Don't need to enforce any constraints for pre-existing evaluations (since we already have the objective value)
+        if projections:
+            xp = dykstra(projections,x0,max_iter=params("dykstra.max_iters"),tol=params("dykstra.d_tol"))
+            if not np.allclose(xp,x0):
+                warnings.warn("x0 not feasible w.r.t given constraints, adjusting", RuntimeWarning)
+                x0 = xp.copy()
+
+        # Enforce lower & upper bounds on x0
+        idx = (x0 < xl)
+        if np.any(idx):
+            warnings.warn("x0 below lower bound, adjusting", RuntimeWarning)
+        x0[idx] = xl[idx]
+
+        idx = (x0 > xu)
+        if np.any(idx):
+            warnings.warn("x0 above upper bound, adjusting", RuntimeWarning)
+        x0[idx] = xu[idx]
 
     # Call main solver (first time)
     diagnostic_info = DiagnosticInfo()

{dfo_ls-1.5.3 → dfo_ls-1.6}/dfols/util.py

@@ -26,13 +26,14 @@ alternative licensing.
 from __future__ import absolute_import, division, print_function, unicode_literals
 
 import logging
+import math
 import numpy as np
 import scipy.linalg as LA
 import sys
 
 
 __all__ = ['sumsq', 'eval_least_squares_with_regularisation', 'model_value', 'random_orthog_directions_within_bounds',
-           'random_directions_within_bounds', 'apply_scaling', 'remove_scaling', 'pbox', 'pball', 'dykstra', 'qr_rank']
+           'random_directions_within_bounds', 'apply_scaling', 'remove_scaling', 'pbox', 'pball', 'dykstra', 'qr_rank', 'replace_nan_with_none']
 
 module_logger = logging.getLogger(__name__) 
 
@@ -268,3 +269,15 @@ def qr_rank(A,tol=1e-15):
     D = np.abs(np.diag(R))
     rank = np.sum(D > tol) 
     return rank, D
+
+
+def replace_nan_with_none(d):
+    # Replace Nan values in a dict/list with None (used for JSON serializing of OptimResults object)
+    if isinstance(d, dict):
+        return {k: replace_nan_with_none(v) for k, v in d.items()}
+    elif isinstance(d, list):
+        return [replace_nan_with_none(i) for i in d]
+    elif isinstance(d, float) and math.isnan(d):
+        return None
+    else:
+        return d