pyGSTi 0.9.12__cp39-cp39-win_amd64.whl → 0.9.13__cp39-cp39-win_amd64.whl

pygsti/optimize/wildcardopt.py

@@ -17,6 +17,25 @@ import numpy as _np
 from pygsti.objectivefns.wildcardbudget import update_circuit_probs as _update_circuit_probs
 from pygsti.optimize.optimize import minimize as _minimize
 
+"""Developer notes
+
+Removed functions
+-----------------
+
+    This file used to have three algorithms for optimizing wildcard budgets that relied on
+    CVXOPT's nonlinear optimization interface. In June 2024 we investigated whether these
+    algorithms could be re-implemented to rely only on CVXPY's modeling capabilities. We
+    came to the conclusion that while that may have been possible, it would have involved
+    an inordinate amount of work, and that for the sake of maintainability it was better to
+    remove these CVXOPT-based algorithms from pyGSTi altogether.
+
+    Here's a hash for one of the last commits on pyGSTi's develop branch that had these
+    algorithms: 723cd24aec3b90d28b0fcd9b31145b920c256acf.
+
+    See https://github.com/sandialabs/pyGSTi/pull/444 for more information.
+
+"""
+
 
 def optimize_wildcard_budget_neldermead(budget, L1weights, wildcard_objfn, two_dlogl_threshold,
                                         redbox_threshold, printer, smart_init=True, max_outer_iters=10,
@@ -48,19 +67,6 @@ def optimize_wildcard_budget_neldermead(budget, L1weights, wildcard_objfn, two_d
 
         return max(0, two_dlogl - two_dlogl_threshold) + percircuit_penalty
 
-    ##For debugging wildcard (see below for suggested insertion point)
-    #def _wildcard_fit_criteria_debug(wv):
-    #    dlogl_elements = logl_wildcard_fn.lsvec(wv)**2  # b/c WC fn only has sqrt of terms implemented now
-    #    for i in range(num_circuits):
-    #        dlogl_percircuit[i] = _np.sum(dlogl_elements[layout.indices_for_index(i)], axis=0)
-    #    two_dlogl_percircuit = 2 * dlogl_percircuit
-    #    two_dlogl = sum(two_dlogl_percircuit)
-    #    print("Aggregate penalty = ", two_dlogl, "-", two_dlogl_threshold, "=", two_dlogl - two_dlogl_threshold)
-    #    print("Per-circuit (redbox) penalty = ", sum(_np.clip(two_dlogl_percircuit - redbox_threshold, 0, None)))
-    #    print(" per-circuit threshold = ", redbox_threshold, " highest violators = ")
-    #    sorted_percircuit = sorted(enumerate(two_dlogl_percircuit), key=lambda x: x[1], reverse=True)
-    #    print('\n'.join(["(%d) %s: %g" % (i, layout.circuits[i].str, val) for i, val in sorted_percircuit[0:10]]))
-
     num_iters = 0
     wvec_init = budget.to_vector()
 
@@ -220,44 +226,6 @@ def _get_critical_circuit_budgets(objfn, redbox_threshold):
     return global_critical_percircuit_budgets
 
 
-# Aggregate 2-delta-logl criteria (for cvxopt call below, as we want this function to be <= 0)
-#  - for each circuit, we have the sum of -2Nf*logl(p) + const. terms
-#  - the derivatives taken below are complicated because they're derivatives with respect to
-#     the circuit's *wildcard budget*, which is effectively w.r.t `p` except all the p's must
-#     sum to 1.  We compute these derivatives as follows:
-#
-#    - 1st deriv: the first derivative of each term is -Nf/p and N is common to all the terms of
-#      a single circuit so this is dictated by chi = f/p >= 0.  All these terms are positive (the
-#      deriv is negative), and we want to move probability from the terms with smallest chi to
-#      largest chi.  Note here that positive `p` means *more* wildcard budget and so the largest-chi
-#      terms have their p_i increase (dp_i = dp) whereas the smallest-chi terms have p_i decrease
-#      (dp_i = -dp).  When multiple terms have the same chi then we split the total dp
-#      (delta-probability) according to 1 / 2nd-deriv = p**2/Nf.  This is so that if
-#      chi1 = f1/p1 = chi2 = f2/p2 and we want the chi's to remain equal after
-#      p1 -> p1 + lambda1*dp, p2 -> p2 + lambda2*dp then we get:
-#      (p1 + lambda1*dp) / f1 = 1/chi1 + lambda1/f1 * dp = 1/chi2 + lambda2/f2 * dp, so
-#      lambda1/f1 = lambda2/f2 => lambda1/lambda2 = f1/f2.  Since lambda1 + lambda2 = 1,
-#      we get lambda1 (1 + f2/f1) = 1 => lambda1 = f1 / (f1 + f2)
-#      In general, lambda_i = f_i / sum_fs_with_max_chi.
-#      Note: f1/p1 = f2/p2 => f1/f2 = p1/p2 so lambda_i also could be = p_i / sum_ps_with_max_chi
-#      We could also derive by wanting the derivs wrt chi be equal:
-#       d(chi1)/dp = d(chi2)/dp => -f1/p1**2 * lambda_1 = -f2/p2**2 * lambda_2
-#       => lambda1/lambda2 = p1/p2 as before (recall dp1 = lambda1 * dp)
-#      Note that this also means the lambdas could be weighted by the full 2nd deriv: Nf/p**2
-#      ** IN SUMMARY, the total derivative is:
-#           -2N * (sum_max_chi(f_i/p_i * lambda_i) - sum_min_chi(f_i/p_i * lambda_i))
-#           = -2N * (max_chi - min_chi)
-#
-#    - 2nd deriv: same as above, but now different lambda_i matter:
-#         = 2N * (sum_max_chi(f_i/p_i**2 * lambda_i**2) - sum_min_chi(f_i/p_i**2 * lambda_i**2))
-#         (where we take the lambda_i as given by the frequencies, so they aren't diff'd)
-#      If we took lambda_i = p_i / sum_of_ps then we'd get:
-#      d/dp (f_i/p_i * lambda_i) = -f_i/p_i**2 * lambda_i**2 + f_i/p_i * dlambda_i/dp
-#                                = -f_i/p_i**2 * lambda_i**2 (see below)
-#      Note dlambda_i/dp = lambda_i / sum_of_ps - p_i / (sum_ps)**2 * sum(lambda_i) = 0
-#      So we get the same result.
-
-
 def _agg_dlogl(current_probs, objfn, two_dlogl_threshold):
     #Note: current_probs is a *local* quantity
     p, f, n, N = current_probs, objfn.freqs, objfn.counts, objfn.total_counts
@@ -369,60 +337,6 @@ def _agg_dlogl_hessian(current_probs, objfn, percircuit_budget_deriv, probs_deri
     return objfn.layout.allsum_local_quantity('c', local_H, use_shared_mem=False)
 
 
-def _proxy_agg_dlogl(x, tvds, fn0s, percircuit_budget_deriv, two_dlogl_threshold):
-    # expects percircuit_budget_deriv to be for all (*global*) circuits
-    percircuit_budgets = _np.dot(percircuit_budget_deriv, x)
-    num_circuits = percircuit_budgets.shape[0]
-    a = 4; b = 2  # fit params: must be same in all proxy fns
-
-    f = 0
-    for i in range(num_circuits):
-        fn0 = fn0s[i]; tvd = tvds[i]; x = percircuit_budgets[i]
-        f += (fn0 / _np.exp(a)) * _np.exp(a - b * (x / tvd)**2 - _np.sqrt(2 * b) * (x / tvd))
-    return f - two_dlogl_threshold
-
-
-def _proxy_agg_dlogl_deriv(x, tvds, fn0s, percircuit_budget_deriv):
-    # expects percircuit_budget_deriv to be for all (*global*) circuits
-    percircuit_budgets = _np.dot(percircuit_budget_deriv, x)
-    num_circuits = percircuit_budgets.shape[0]
-    a = 4; b = 2  # fit params: must be same in all proxy fns
-
-    agg_dlogl_deriv_wrt_percircuit_budgets = _np.zeros(num_circuits, 'd')
-    for i in range(num_circuits):
-        fn0 = fn0s[i]; tvd = tvds[i]; x = percircuit_budgets[i]
-        agg_dlogl_deriv_wrt_percircuit_budgets[i] = \
-            (fn0 / _np.exp(a)) * _np.exp(a - b * (x / tvd)**2
-                                         - _np.sqrt(2 * b) * (x / tvd)) * (-2 * b * x / tvd**2
-                                                                           - _np.sqrt(2 * b) / tvd)
-    #This isn't always true in "proxy" case - maybe clip to 0?
-    #assert(_np.all(agg_dlogl_deriv_wrt_percircuit_budgets <= 0)), \
-    #    "Derivative of aggregate LLR wrt any circuit budget should be negative"
-    return _np.dot(agg_dlogl_deriv_wrt_percircuit_budgets, percircuit_budget_deriv)
-
-
-def _proxy_agg_dlogl_hessian(x, tvds, fn0s, percircuit_budget_deriv):
-    # expects percircuit_budget_deriv to be for all (*global*) circuits
-    percircuit_budgets = _np.dot(percircuit_budget_deriv, x)
-    num_circuits = percircuit_budgets.shape[0]
-    a = 4; b = 2  # fit params: must be same in all proxy fns
-
-    agg_dlogl_hessian_wrt_percircuit_budgets = _np.zeros(num_circuits)
-    for i in range(num_circuits):
-        fn0 = fn0s[i]; tvd = tvds[i]; x = percircuit_budgets[i]
-        agg_dlogl_hessian_wrt_percircuit_budgets[i] = \
-            (fn0 / _np.exp(a)) * _np.exp(a - b * (x / tvd)**2 - _np.sqrt(2 * b) * (x / tvd)) * (
-                (-2 * b * x / tvd**2 - _np.sqrt(2 * b) / tvd)**2 - 2 * b / tvd**2)
-    assert(_np.all(agg_dlogl_hessian_wrt_percircuit_budgets >= -1e-8)), \
-        "Hessian of aggregate LLR wrt any circuit budget should be positive"
-    H = _np.dot(percircuit_budget_deriv.T,
-                _np.dot(_np.diag(agg_dlogl_hessian_wrt_percircuit_budgets),
-                        percircuit_budget_deriv))   # (nW, nC)(nC)(nC, nW)
-    #evals = _np.linalg.eigvals(H)
-    #assert(_np.all(evals >= -1e-8))
-    return H
-
-
 def _get_percircuit_budget_deriv(budget, layout):
     """ Returns local_percircuit_budget_deriv, global_percircuit_budget_deriv """
     percircuit_budget_deriv = budget.precompute_for_same_circuits(layout.circuits)  # for *local* circuits
@@ -492,168 +406,6 @@ def optimize_wildcard_bisect_alpha(budget, objfn, two_dlogl_threshold, redbox_th
     return
 
 
-def optimize_wildcard_budget_cvxopt(budget, L1weights, objfn, two_dlogl_threshold, redbox_threshold,
-                                    printer, abs_tol=1e-5, rel_tol=1e-5, max_iters=50):
-    """Uses CVXOPT to optimize the wildcard budget.  Includes both aggregate and per-circuit constraints."""
-    #Use cvxopt
-    import cvxopt as _cvxopt
-    # Minimize f_0(wv) = |wv|_1 (perhaps weighted) subject to the constraints:
-    #  dot(percircuit_budget_deriv, wv) >= critical_percircuit_budgets
-    #  2 * aggregate_dlogl <= two_dlogl_threshold  => f_1(wv) = 2 * aggregate_dlogl(wv) - threshold <= 0
-
-    layout = objfn.layout
-    wv = budget.to_vector().copy()
-    n = len(wv)
-    x0 = wv.reshape((n, 1))  # TODO - better guess?
-
-    initial_probs = objfn.probs.copy()  # *local*
-    current_probs = initial_probs.copy()
-    percircuit_budget_deriv, global_percircuit_budget_deriv = _get_percircuit_budget_deriv(budget, layout)
-
-    critical_percircuit_budgets = _get_critical_circuit_budgets(objfn, redbox_threshold)  # for *global* circuits
-    critical_percircuit_budgets.shape = (len(critical_percircuit_budgets), 1)
-
-    _cvxopt.solvers.options['abstol'] = abs_tol
-    _cvxopt.solvers.options['reltol'] = rel_tol
-    _cvxopt.solvers.options['maxiters'] = max_iters
-
-    def F(x=None, z=None, debug=True):
-        if z is None and x is None:
-            # (m, x0) where m is number of nonlinear constraints and x0 is in domain of f
-            return (1, _cvxopt.matrix(x0))
-
-        if min(x) < 0.0:
-            return None  # don't allow negative wildcard vector components
-
-        budget.from_vector(_np.array(x))
-        p_deriv = budget.update_probs(initial_probs, current_probs, objfn.freqs, layout, percircuit_budget_deriv,
-                                      return_deriv=True)
-
-        #Evaluate F(x) => return (f, Df)
-        f = _cvxopt.matrix(_np.array([_agg_dlogl(current_probs, objfn,
-                                                 two_dlogl_threshold)]).reshape((1, 1)))  # shape (m,1)
-        Df = _cvxopt.matrix(_np.empty((1, n), 'd'))  # shape (m, n)
-        Df[0, :] = _agg_dlogl_deriv(current_probs, objfn, percircuit_budget_deriv, p_deriv)
-
-        if z is None:
-            return f, Df
-
-        # additionally, compute H = z_0 * Hessian(f_0)(wv)
-        H = _cvxopt.matrix(z[0] * _agg_dlogl_hessian(current_probs, objfn, percircuit_budget_deriv, p_deriv))
-        evals = _np.linalg.eigvals(H)
-        assert(_np.all(evals >= -1e-8))  # tests *global* H
-        return f, Df, H
-
-    #check_fd([0.0001] * n, True)
-
-    #CVXOPT
-    printer.log("Beginning cvxopt.cpl solve...")
-    c = _cvxopt.matrix(L1weights.reshape((n, 1)))
-    G = -_cvxopt.matrix(_np.concatenate((global_percircuit_budget_deriv, _np.identity(n, 'd')), axis=0))
-    h = -_cvxopt.matrix(_np.concatenate((critical_percircuit_budgets, _np.zeros((n, 1), 'd')), axis=0))
-    #result = _cvxopt.solvers.cpl(c, F)  # kktsolver='ldl2'
-    result = _cvxopt.solvers.cpl(c, F, G, h)  # kktsolver='ldl2'
-
-    #This didn't seem to help much:
-    #print("Attempting restart...")
-    #x0[:,0] = list(result['x'])
-    #result = _cvxopt.solvers.cpl(c, F) # kktsolver='ldl2'
-
-    printer.log("CVXOPT result = " + str(result))
-    printer.log("x = " + str(list(result['x'])))
-    printer.log("y = " + str(list(result['y'])))
-    printer.log("znl = " + str(list(result['znl'])))
-    printer.log("snl = " + str(list(result['snl'])))
-    budget.from_vector(result['x'])
-    return
-
-
-def optimize_wildcard_budget_cvxopt_zeroreg(budget, L1weights, objfn, two_dlogl_threshold, redbox_threshold,
-                                            printer, abs_tol=1e-5, rel_tol=1e-5, max_iters=50, small=1e-6):
-    """Adds regularization of the L1 term around zero values of the budget.  This doesn't seem to help much."""
-    #Use cvxopt
-    import cvxopt as _cvxopt
-    # Minimize f_0(wv) = |wv|_1 (perhaps weighted) subject to the constraints:
-    #  dot(percircuit_budget_deriv, wv) >= critical_percircuit_budgets
-    #  2 * aggregate_dlogl <= two_dlogl_threshold  => f_1(wv) = 2 * aggregate_dlogl(wv) - threshold <= 0
-
-    layout = objfn.layout
-    wv = budget.to_vector().copy()
-    n = len(wv)
-    x0 = wv.reshape((n, 1))
-    c = L1weights.reshape((n, 1))
-    SMALL2 = small**2
-
-    initial_probs = objfn.probs.copy()
-    current_probs = initial_probs.copy()
-    percircuit_budget_deriv, global_percircuit_budget_deriv = _get_percircuit_budget_deriv(budget, layout)
-
-    critical_percircuit_budgets = _get_critical_circuit_budgets(objfn, redbox_threshold)
-    critical_percircuit_budgets.shape = (len(critical_percircuit_budgets), 1)
-    assert(_np.all(critical_percircuit_budgets >= 0))
-    assert(_np.all(percircuit_budget_deriv >= 0))
-
-    _cvxopt.solvers.options['abstol'] = abs_tol
-    _cvxopt.solvers.options['reltol'] = rel_tol
-    _cvxopt.solvers.options['maxiters'] = max_iters
-
-    def F(x=None, z=None):
-        if z is None and x is None:
-            # (m, x0) where m is number of nonlinear constraints and x0 is in domain of f
-            return (1, _cvxopt.matrix(x0))
-
-        if min(x) < 0.0:
-            return None  # don't allow negative wildcard vector components
-
-        budget.from_vector(x)
-        p_deriv = budget.update_probs(initial_probs, current_probs, objfn.freqs, layout, percircuit_budget_deriv,
-                                      return_deriv=True)
-
-        #Evaluate F(x) => return (f, Df)
-        sqrtVec = _np.sqrt((c * x)**2 + SMALL2)
-        f = _cvxopt.matrix(_np.array([float(_np.sum(sqrtVec)),
-                                      _agg_dlogl(current_probs, objfn,
-                                                 two_dlogl_threshold)]).reshape((2, 1)))  # shape (m+1,1)
-
-        L1term_grad = c if SMALL2 == 0.0 else c**2 * x / sqrtVec
-        Df = _cvxopt.matrix(_np.empty((2, n), 'd'))  # shape (m+1, n)
-        Df[0, :] = L1term_grad[:, 0]
-        Df[1, :] = _agg_dlogl_deriv(current_probs, objfn, percircuit_budget_deriv, p_deriv)
-        #print("rank Df=", _np.linalg.matrix_rank(Df))
-        if z is None:
-            return f, Df
-
-        # additionally, compute H = z_0 * Hessian(f_0)(wv) + z_1 * Hessian(f_1)(wv)
-        L1_term_hess = _np.zeros((n, n), 'd') if SMALL2 == 0.0 else \
-            _np.diag(-1.0 / (sqrtVec**3) * (c**2 * x)**2 + c**2 / sqrtVec)
-        Hf = _cvxopt.matrix(z[0] * L1_term_hess + z[1] * _agg_dlogl_hessian(current_probs, objfn,
-                                                                            percircuit_budget_deriv, p_deriv))
-        #print("rank Hf=", _np.linalg.matrix_rank(Hf), " z[1]=",z[1])
-        return f, Df, Hf
-
-    #CVXOPT
-    printer.log("Beginning cvxopt.cp solve...")
-    #print("Rank G = ",_np.linalg.matrix_rank(percircuit_budget_deriv))
-    #result = _cvxopt.solvers.cp(F)
-    # Condition is Gx <= h => -Gx >= -h
-    G = -_cvxopt.matrix(_np.concatenate((global_percircuit_budget_deriv, _np.identity(n, 'd')), axis=0))
-    h = -_cvxopt.matrix(_np.concatenate((critical_percircuit_budgets, _np.zeros((n, 1), 'd')), axis=0))
-    result = _cvxopt.solvers.cp(F, G, h)
-
-    #This didn't seem to help much:
-    #print("Attempting restart...")
-    #x0[:,0] = list(result['x'])
-    #result = _cvxopt.solvers.cpl(c, F) # kktsolver='ldl2'
-
-    printer.log("CVXOPT result = " + str(result))
-    printer.log("x = " + str(list(result['x'])))
-    printer.log("y = " + str(list(result['y'])))
-    printer.log("znl = " + str(list(result['znl'])))
-    printer.log("snl = " + str(list(result['snl'])))
-    budget.from_vector(result['x'])
-    return
-
-
 def optimize_wildcard_budget_barrier(budget, L1weights, objfn, two_dlogl_threshold,
                                      redbox_threshold, printer, tol=1e-7, max_iters=50, num_steps=3,
                                      save_debugplot_data=False):
@@ -776,13 +528,6 @@ def optimize_wildcard_budget_barrier(budget, L1weights, objfn, two_dlogl_thresho
                 Hobj = t * _np.diag(-1.0 / (sqrtVec**3) * (c**2 * x)**2 + c**2 / sqrtVec) + Hbarrier
             return obj, Dobj, Hobj
 
-        #import scipy.optimize
-        #def barrier_obj(x):
-        #    x = _np.clip(x, 1e-10, None)
-        #    return t * _np.dot(c.T, x) - _np.log(-barrierF(x, False))
-        #result = scipy.optimize.minimize(barrier_obj, x, method="CG")
-        #x = _np.clip(result.x, 0, None)
-
         x, debug_x_list = NewtonSolve(x, NewtonObjective, NewtonObjective_derivs, tol, max_iters, printer - 1)
         #x, debug_x_list = NewtonSolve(x, NewtonObjective, None, tol, max_iters, printer - 1)  # use finite-diff derivs
 
@@ -924,86 +669,6 @@ def NewtonSolve(initial_x, fn, fn_with_derivs=None, dx_tol=1e-6, max_iters=20, p
     return x, x_list
 
 
-def optimize_wildcard_budget_cvxopt_smoothed(budget, L1weights, objfn, two_dlogl_threshold, redbox_threshold,
-                                             printer, abs_tol=1e-5, rel_tol=1e-5, max_iters=50):
-    """
-    Uses a smooted version of the objective function.  Doesn't seem to help much.
-
-    The thinking here was to eliminate the 2nd derivative discontinuities of the original problem.
-    """
-    import cvxopt as _cvxopt
-
-    layout = objfn.layout
-    wv = budget.to_vector().copy()
-    n = len(wv)
-    x0 = wv.reshape((n, 1))  # TODO - better guess?
-
-    #initial_probs = objfn.probs.copy()
-    #current_probs = initial_probs.copy()
-    percircuit_budget_deriv, global_percircuit_budget_deriv = _get_percircuit_budget_deriv(budget, layout)
-    critical_percircuit_budgets = _get_critical_circuit_budgets(objfn, redbox_threshold)
-    critical_percircuit_budgets.shape = (len(critical_percircuit_budgets), 1)
-    num_circuits = len(layout.circuits)
-
-    _cvxopt.solvers.options['abstol'] = abs_tol
-    _cvxopt.solvers.options['reltol'] = rel_tol
-    _cvxopt.solvers.options['maxiters'] = max_iters
-
-    #Prepare for proxy_barrierF evaluations
-    local_tvds = _np.zeros(num_circuits, 'd')
-    local_fn0s = _np.zeros(num_circuits, 'd')
-    for i in range(num_circuits):
-        p = objfn.probs[layout.indices_for_index(i)]
-        f = objfn.freqs[layout.indices_for_index(i)]
-        nn = objfn.counts[layout.indices_for_index(i)]  # don't re-use 'n' variable!
-        N = objfn.total_counts[layout.indices_for_index(i)]
-        dlogl_elements = objfn.raw_objfn.terms(p, nn, N, f)  # N * f * _np.log(f / p)
-        local_fn0s[i] = 2 * _np.sum(dlogl_elements)
-        local_tvds[i] = 0.5 * _np.sum(_np.abs(p - f))
-    tvds = layout.allgather_local_array('c', local_tvds)
-    fn0s = layout.allgather_local_array('c', local_fn0s)
-
-    def F(x=None, z=None, debug=True):
-        if z is None and x is None:
-            # (m, x0) where m is number of nonlinear constraints and x0 is in domain of f
-            return (1, _cvxopt.matrix(x0))
-
-        if min(x) < 0.0:
-            return None  # don't allow negative wildcard vector components
-
-        #budget.from_vector(_np.array(x))
-        #budget.update_probs(initial_probs, current_probs, objfn.freqs, layout, percircuit_budget_deriv)
-
-        #Evaluate F(x) => return (f, Df)
-        f = _cvxopt.matrix(_np.array([_proxy_agg_dlogl(x, tvds, fn0s, global_percircuit_budget_deriv,
-                                                       two_dlogl_threshold)]).reshape((1, 1)))  # shape (m,1)
-        Df = _cvxopt.matrix(_np.empty((1, n), 'd'))  # shape (m, n)
-        Df[0, :] = _proxy_agg_dlogl_deriv(x, tvds, fn0s, global_percircuit_budget_deriv)
-
-        if z is None:
-            return f, Df
-
-        # additionally, compute H = z_0 * Hessian(f_0)(wv)
-        H = _cvxopt.matrix(z[0] * _proxy_agg_dlogl_hessian(x, tvds, fn0s, global_percircuit_budget_deriv))
-        evals = _np.linalg.eigvals(H)
-        assert(_np.all(evals >= -1e-8))
-        return f, Df, H
-
-    printer.log("Beginning cvxopt.cpl solve with smoothed (proxy) fn...")
-    c = _cvxopt.matrix(L1weights.reshape((n, 1)))
-    G = -_cvxopt.matrix(_np.concatenate((global_percircuit_budget_deriv, _np.identity(n, 'd')), axis=0))
-    h = -_cvxopt.matrix(_np.concatenate((critical_percircuit_budgets, _np.zeros((n, 1), 'd')), axis=0))
-    result = _cvxopt.solvers.cpl(c, F, G, h)  # kktsolver='ldl2'
-
-    printer.log("CVXOPT result = " + str(result))
-    printer.log("x = " + str(list(result['x'])))
-    printer.log("y = " + str(list(result['y'])))
-    printer.log("znl = " + str(list(result['znl'])))
-    printer.log("snl = " + str(list(result['snl'])))
-    budget.from_vector(result['x'])
-    return
-
-
 def _compute_fd(x, fn, compute_hessian=True, eps=1e-7):
     x_len = len(x)
     grad = _np.zeros(x_len, 'd')
@@ -1026,247 +691,3 @@ def _compute_fd(x, fn, compute_hessian=True, eps=1e-7):
             f_eps_kl = fn(x_eps_kl)
             hess[k, l] = (f_eps_kl - f_eps_k - f_eps_l + f0) / eps**2
     return grad, hess
-
-
-#DEBUG: check with finite diff derivatives:
-#def _check_fd(wv_base, chk_hessian=False):
-#    wv_base = _np.array(wv_base, 'd')  # [0.0001]*3
-#    wv_len = len(wv_base)
-#    grad = _np.zeros(wv_len, 'd')
-#    f0, grad_chk = F(wv_base, debug=False)
-#    eps = 1e-7
-#    for k in range(len(wv_base)):
-#        wv_eps = wv_base.copy(); wv_eps[k] += eps
-#        f_eps, _ = F(wv_eps, debug=False)
-#        grad[k] = (f_eps[0] - f0[0]) / eps
-#    rel_diff_norm = _np.linalg.norm(grad - grad_chk) / _np.linalg.norm(grad)
-#    #print("GRAD CHECK:")
-#    #print(grad)
-#    #print(grad_chk)
-#    #print("  diff = ",grad - grad_chk, " rel_diff_norm=", rel_diff_norm)
-#    print("GRAD CHK ", rel_diff_norm)
-#    assert(rel_diff_norm < 1e-3)
-#    if chk_hessian is False: return
-#
-#    hess = _np.zeros((wv_len, wv_len), 'd')
-#    f0, _, H_chk = F(wv_base, [1.0], debug=False)
-#    eps = 1e-7
-#    for k in range(wv_len):
-#        wv_eps_k = wv_base.copy(); wv_eps_k[k] += eps
-#        f_eps_k, _ = F(wv_eps_k, debug=False)
-#        for l in range(wv_len):
-#            wv_eps_l = wv_base.copy(); wv_eps_l[l] += eps
-#            f_eps_l, _ = F(wv_eps_l, debug=False)
-#            wv_eps_kl = wv_eps_k.copy(); wv_eps_kl[l] += eps
-#            f_eps_kl, _ = F(wv_eps_kl, debug=False)
-#            hess[k, l] = (f_eps_kl[0] - f_eps_k[0] - f_eps_l[0] + f0[0]) / eps**2
-#    rel_diff_norm = _np.linalg.norm(hess - H_chk) / _np.linalg.norm(hess)
-#    #print("HESSIAN CHECK:")
-#    #print(hess)
-#    #print(H_chk)
-#    #print("  diff = ",hess - H_chk, " rel_diff_norm=", rel_diff_norm)
-#    print("HESS CHK ", rel_diff_norm)
-#    #assert(rel_diff_norm < 5e-2)
-
-
-#UNUSED?
-#def _wildcard_objective_firstterms(current_probs):
-#    dlogl_elements = objfn.raw_objfn.terms(current_probs, objfn.counts, objfn.total_counts, objfn.freqs)
-#    for i in range(num_circuits):
-#        dlogl_percircuit[i] = _np.sum(dlogl_elements[layout.indices_for_index(i)], axis=0)
-#
-#    two_dlogl_percircuit = 2 * dlogl_percircuit
-#    two_dlogl = sum(two_dlogl_percircuit)
-#    return max(0, two_dlogl - two_dlogl_threshold) \
-#        + sum(_np.clip(two_dlogl_percircuit - redbox_threshold, 0, None))
-#
-#def _advance_probs(layout, current_probs, dlogl_percircuit, dlogl_delements, delta_percircuit_budgets):
-#    num_circuits = len(layout.circuits)
-#    delta_probs = _np.zeros(len(current_probs), 'd')
-#    for i in range(num_circuits):
-#        #if 2 * dlogl_percircuit[i] <= redbox_threshold and global_criteria_met: continue
-#
-#        step = delta_percircuit_budgets[i]
-#        #p = current_probs[layout.indices_for_index(i)]
-#        chis = dlogl_delements[layout.indices_for_index(i)]
-#        maxes = _np.array(_np.abs(chis - _np.max(chis)) < 1.e-4, dtype=int)
-#        mins = _np.array(_np.abs(chis - _np.min(chis)) < 1.e-4, dtype=int)
-#        add_to = step * mins / sum(mins)
-#        take_from = step * maxes / sum(maxes)
-#        delta_probs[layout.indices_for_index(i)] = add_to - take_from
-#    return delta_probs
-#
-#
-#def wildcard_probs_propagation(budget, initial_wv, final_wv, objfn, layout, num_steps=10):
-#    #Begin with a zero budget
-#    current_probs = objfn.probs.copy()
-#
-#    percircuit_budget_deriv = budget.precompute_for_same_circuits(layout.circuits)
-#    dlogl_percircuit = objfn.percircuit()
-#
-#    num_circuits = len(layout.circuits)
-#    assert(len(dlogl_percircuit) == num_circuits)
-#
-#    delta_wv = (final_wv - initial_wv) / num_steps
-#    wv = initial_wv.copy()
-#    for i in range(nSteps):
-#        wv += delta_wv
-#        dlogl_elements = objfn.raw_objfn.terms(current_probs, objfn.counts, objfn.total_counts, objfn.freqs)
-#        for i in range(num_circuits):
-#            dlogl_percircuit[i] = _np.sum(dlogl_elements[layout.indices_for_index(i)], axis=0)
-#        dlogl_delements = objfn.raw_objfn.dterms(current_probs, objfn.counts, objfn.total_counts, objfn.freqs)
-#
-#        two_dlogl = sum(2 * dlogl_percircuit)
-#        perbox_residual = sum(_np.clip(2 * dlogl_percircuit - redbox_threshold, 0, None))
-#        print("Advance: global=", two_dlogl - two_dlogl_threshold, " percircuit=", perbox_residual)
-#        print("  wv=", wv)
-#
-#        delta_percircuit_budgets = _np.dot(percircuit_budget_deriv, delta_wv)
-#        delta_probs = _advance_probs(layout, current_probs, dlogl_percircuit,
-#                                     dlogl_delements, delta_percircuit_budgets)  # updates current_probs
-#        print("|delta probs| = ", _np.linalg.norm(delta_probs))
-#        current_probs += delta_probs
-#    return currrent_probs
-#def wildcard_opt_by_propagation()  #TODO
-#    # Time-evolution approach:  Walk downhill in steps until constraints ("firstterms") are satisfied
-#    #wv = budget.to_vector().copy()
-#
-#    def _criteria_deriv(current_probs, dlogl_percircuit, dlogl_delements, mode, global_criteria_met):
-#        # derivative of firstterms wrt per-circuit wilcard budgets - namely if that budget goes up how to most
-#        # efficiently reduce firstterms
-#        # in doing so, this computes how the per-circuit budget should be allocated to probabilities
-#        # (i.e. how probs should be updated) to achieve this decrease in firstterms
-#        ret = _np.zeros(num_circuits)
-#        max_delta = _np.zeros(num_circuits)  # maximum amount of change in per-circuit budget before hitting a
-#        #   discontinuity in 2nd deriv
-#        for i in range(num_circuits):
-#            if mode == "percircuit" and 2 * dlogl_percircuit[i] <= redbox_threshold:
-#                continue  # don't include this circuit's contribution
-#            elif mode == "aggregate":  # all circuits contribute
-#                prefactor = 1.0
-#            else:  # mode == "both"
-#                prefactor = 2.0  # contributes twice: once for per-circuit and once for aggregate
-#                if 2 * dlogl_percircuit[i] <= redbox_threshold:
-#                    if global_criteria_met: continue  # no contribution at all_circuits_needing_data
-#                    else: prefactor = 1.0
-#
-#            chis = dlogl_delements[layout.indices_for_index(i)]  # ~ f/p  (deriv of f*log(p))
-#            highest_chi, lowest_chi = _np.max(chis), _np.min(chis)
-#            bmaxes = _np.array(_np.abs(chis - highest_chi) < 1.e-4, dtype=bool)
-#            bmins = _np.array(_np.abs(chis - lowest_chi) < 1.e-4, dtype=bool)
-#            maxes = _np.array(_np.abs(chis - _np.max(chis)) < 1.e-4, dtype=int)
-#            mins = _np.array(_np.abs(chis - _np.min(chis)) < 1.e-4, dtype=int)
-#
-#            next_chis = chis.copy(); next_chis[bmaxes] = 1.0; next_chis[bmins] = 1.0
-#            #p = current_probs[layout.indices_for_index(i)]
-#            f = objfn.freqs[layout.indices_for_index(i)]
-#            next_highest_chi = _np.max(next_chis)  # 2nd highest chi value (may be duplicated)
-#            next_lowest_chi = _np.min(next_chis)  # 2nd lowest chi value (may be duplicated)
-#
-#            # 1/chi = p/f, (1/chi'-1/chi) = dp/f => dp = f(chi - chi')/(chi chi')
-#            delta_p = _np.zeros(chis.shape, 'd')
-#            delta_p[bmaxes] = f[bmaxes] * (1. / chis[bmaxes] - 1 / next_highest_chi)
-#            delta_p[bmins] = f[bmins] * (1. / chis[bmins] - 1 / next_lowest_chi)
-#            max_delta[i] = _np.max(_np.abs(delta_p))
-#
-#            ret[i] = prefactor * _np.sum(chis * (mins / sum(mins) - maxes / sum(maxes)))
-#        return ret, max_delta
-#
-#
-#    for mode in (): #("both",): #("percircuit", "aggregate"):  # how many & which criteria to enforce on each pass.
-#        print("Stage w/mode = ",mode)
-#        step = 0.01
-#        itr = 0
-#        L1grad = L1weights
-#        imax = None
-#        last_objfn_value = None; last_probs = None  # DEBUG
-#        last_dlogl_percircuit = last_dlogl_elements = None # DEBUG
-#        while True:
-#
-#            #Compute current log-likelihood values and derivates wrt probabilities
-#            dlogl_elements = objfn.raw_objfn.terms(current_probs, objfn.counts, objfn.total_counts, objfn.freqs)
-#            for i in range(num_circuits):
-#                dlogl_percircuit[i] = _np.sum(dlogl_elements[layout.indices_for_index(i)], axis=0)
-#            dlogl_delements = objfn.raw_objfn.dterms(current_probs, objfn.counts, objfn.total_counts, objfn.freqs)
-#            two_dlogl_percircuit = 2 * dlogl_percircuit
-#            two_dlogl = sum(two_dlogl_percircuit)
-#            global_criteria_met = two_dlogl < two_dlogl_threshold
-#
-#            # check aggregate and per-circuit criteria - exit if met
-#            if mode == "aggregate":
-#                objfn_value = max(two_dlogl - two_dlogl_threshold, 0)
-#            elif mode == "percircuit":
-#                perbox_residual = sum(_np.clip(two_dlogl_percircuit - redbox_threshold, 0, None))
-#                objfn_value = perbox_residual
-#            elif mode == "both":
-#                objfn_value = max(two_dlogl - two_dlogl_threshold, 0) \
-#                    + sum(_np.clip(two_dlogl_percircuit - redbox_threshold, 0, None))
-#
-#            print("Iter ", itr, ": mode=", mode, " objfn=", objfn_value, " moved in", imax)
-#            print("  wv=", wv); itr += 1
-#            if objfn_value < 1e-10: # if global_criteria_met and perbox_residual < 1e-10:
-#                break  # DONE!
-#            if last_objfn_value is not None and last_objfn_value < objfn_value:
-#                iproblem = _np.argmax(dlogl_percircuit - last_dlogl_percircuit)
-#                print("Circuit  ",iproblem," dlogl=", last_dlogl_percircuit[iproblem], " => ",
-#                      dlogl_percircuit[iproblem])
-#                print("  probs: ",last_probs[layout.indices_for_index(iproblem)], " => ",
-#                      current_probs[layout.indices_for_index(iproblem)])
-#                print("  freqs: ",objfn.freqs[layout.indices_for_index(iproblem)])
-#                import bpdb; bpdb.set_trace()
-#                assert(False), "Objective function should be monotonic!!!"
-#            last_objfn_value = objfn_value
-#            last_probs = current_probs.copy()
-#            last_dlogl_percircuit = dlogl_percircuit.copy()
-#            last_dlogl_elements = dlogl_elements.copy()
-#
-#            #import bpdb; bpdb.set_trace()
-#            criteria_deriv_wrt_percircuit_budgets, maximum_percircuit_budget_delta = \
-#                _criteria_deriv(current_probs, dlogl_percircuit, dlogl_delements, mode, global_criteria_met)
-#            wv_grad = _np.dot(criteria_deriv_wrt_percircuit_budgets, percircuit_budget_deriv) #+ L1grad
-#            grad_norm = _np.linalg.norm(wv_grad)
-#            assert(grad_norm > 1e-6), \
-#                "Gradient norm == 0! - cannot reduce constraint residuals with more wildcard!"
-#
-#            imax = _np.argmax(_np.abs(wv_grad / L1grad)); sgn = _np.sign(wv_grad[imax])
-#            wv_grad[:] = 0; wv_grad[imax] = sgn
-#            downhill_direction = (-wv_grad / _np.linalg.norm(wv_grad))
-#
-#            #Constant step:
-#            #step = 1e-5
-#            # Variable step: expected reduction = df/dw * dw, so set |dw| = 0.01 * current_f / |df/dw|
-#            #step = (0.01 * objfn_value / grad_norm)
-#
-#            #Step based on next discontinuity ("breakpoint")
-#            # require _np.dot(percircuit_budget_deriv, step * downhill_direction) < maximum_percircuit_budget_delta
-#            step = _np.min(maximum_percircuit_budget_delta / _np.dot(percircuit_budget_deriv, downhill_direction))
-#            assert(step > 0)
-#            step = min(step, 1e-5)  # don't allow too large of a step...
-#
-#            delta_wv = downhill_direction * step
-#            wv += delta_wv
-#
-#            delta_percircuit_budgets = _np.dot(percircuit_budget_deriv, delta_wv)
-#            #assert(_np.all(delta_percircuit_budgets >= 0))
-#            if not _np.all(delta_percircuit_budgets >= 0):
-#                import bpdb; bpdb.set_trace()
-#                pass
-#
-#            delta_probs = _advance_probs(layout, current_probs, dlogl_percircuit, dlogl_delements,
-#                                         delta_percircuit_budgets)  #, global_criteria_met)  # updates current_probs
-#            print("|delta probs| = ", _np.linalg.norm(delta_probs))
-#            current_probs += delta_probs
-#
-#    #assert(False), "STOP"
-#    wv_new = wv
-#    print("NEW TEST - final wildcard is ", wv_new)
-#
-#This didn't work well:
-##Experiment with "soft" min and max functions to see if that fixes cvxopt getting stuck
-## so far, this hasn't helped.
-#
-#def _softmax(ar):
-#    return _np.log(_np.sum([_np.exp(x) for x in ar]))
-#
-#def _softmin(ar):
-#    return -_np.log(_np.sum([_np.exp(-x) for x in ar]))