casadi 3.6.3__cp35-none-win_amd64.whl → 3.6.5__cp35-none-win_amd64.whl

casadi/include/alpaqa/accelerators/anderson.hpp

@@ -0,0 +1,133 @@
+#pragma once
+
+#include <alpaqa/accelerators/internal/anderson-helpers.hpp>
+#include <alpaqa/config/config.hpp>
+#include <alpaqa/export.hpp>
+
+#include <limits>
+#include <stdexcept>
+
+namespace alpaqa {
+
+/// Parameters for the @ref AndersonAccel class.
+template <Config Conf = DefaultConfig>
+struct AndersonAccelParams {
+    USING_ALPAQA_CONFIG(Conf);
+    /// Length of the history to keep (the number of columns in the QR
+    /// factorization).
+    /// If this number is greater than the problem dimension, the memory is set
+    /// to the problem dimension (otherwise the system is underdetermined).
+    length_t memory = 10;
+    /// Minimum divisor when solving close to singular systems,
+    /// scaled by the maximum eigenvalue of R.
+    real_t min_div_fac = real_t(1e2) * std::numeric_limits<real_t>::epsilon();
+};
+
+/**
+ * Anderson Acceleration.
+ *
+ * Algorithm for accelerating fixed-point iterations for finding fixed points
+ * of a function @f$ g @f$, i.e. @f$ g(x^\star) = x^\star @f$, or equivalently,
+ * roots of the residual @f$ r(x) \triangleq g(x) - x @f$.
+ *
+ * @todo   Condition estimation of the QR factorization.
+ *
+ * @ingroup grp_Accelerators
+ */
+template <Config Conf = DefaultConfig>
+class AndersonAccel {
+  public:
+    USING_ALPAQA_CONFIG(Conf);
+    using Params = AndersonAccelParams<config_t>;
+
+    AndersonAccel() = default;
+    /// @param  params
+    ///         Parameters.
+    AndersonAccel(Params params) : params(params) {}
+    /// @param  params
+    ///         Parameters.
+    /// @param  n
+    ///         Problem dimension (size of the vectors).
+    AndersonAccel(Params params, length_t n) : params(params) { resize(n); }
+
+    /// Change the problem dimension. Flushes the history.
+    /// @param  n
+    ///         Problem dimension (size of the vectors).
+    void resize(length_t n) {
+        length_t m_AA = std::min(n, params.memory); // TODO: support m > n?
+        qr.resize(n, m_AA);
+        G.resize(n, m_AA);
+        rₗₐₛₜ.resize(n);
+        γ_LS.resize(m_AA);
+        initialized = false;
+    }
+
+    /// Call this function on the first iteration to initialize the accelerator.
+    void initialize(crvec g_0, crvec r_0) {
+        assert(g_0.size() == n());
+        assert(r_0.size() == n());
+        G.col(0) = g_0;
+        rₗₐₛₜ    = r_0;
+        qr.reset();
+        initialized = true;
+    }
+
+    /// Compute the accelerated iterate @f$ x^k_\text{AA} @f$, given the
+    /// function value at the current iterate @f$ g^k = g(x^k) @f$ and the
+    /// corresponding residual @f$ r^k = g^k - x^k @f$.
+    void compute(crvec gₖ, crvec rₖ, rvec xₖ_aa) {
+        if (!initialized)
+            throw std::logic_error("AndersonAccel::compute() called before "
+                                   "AndersonAccel::initialize()");
+        minimize_update_anderson(qr, G,                             // inout
+                                 rₖ, rₗₐₛₜ, gₖ, params.min_div_fac, // in
+                                 γ_LS, xₖ_aa);                      // out
+        rₗₐₛₜ = rₖ;
+    }
+    /// @copydoc compute(crvec, crvec, rvec)
+    void compute(crvec gₖ, vec &&rₖ, rvec xₖ_aa) {
+        if (!initialized)
+            throw std::logic_error("AndersonAccel::compute() called before "
+                                   "AndersonAccel::initialize()");
+        minimize_update_anderson(qr, G,                             // inout
+                                 rₖ, rₗₐₛₜ, gₖ, params.min_div_fac, // in
+                                 γ_LS, xₖ_aa);                      // out
+        rₗₐₛₜ = std::move(rₖ);
+    }
+
+    /// Reset the accelerator (but keep the last function value and residual, so
+    /// calling @ref initialize is not necessary).
+    void reset() {
+        index_t newest_g_idx = qr.ring_tail();
+        if (newest_g_idx != 0)
+            G.col(0) = G.col(newest_g_idx);
+        qr.reset();
+    }
+
+    /// Get the problem dimension.
+    length_t n() const { return qr.n(); }
+    /// Get the maximum number of stored columns.
+    length_t history() const { return qr.m(); }
+    /// Get the number of columns currently stored in the buffer.
+    length_t current_history() const { return qr.current_history(); }
+
+    /// Get the parameters.
+    const Params &get_params() const { return params; }
+
+    std::string get_name() const {
+        return "AndersonAccel<" + std::string(config_t::get_name()) + '>';
+    }
+
+    /// Scale the factorization
+    void scale_R(real_t scal) { qr.scale_R(scal); }
+
+  private:
+    Params params;
+    LimitedMemoryQR<config_t> qr;
+    mat G;
+    vec rₗₐₛₜ;
+    vec γ_LS;
+    bool initialized = false;
+};
+
+} // namespace alpaqa

casadi/include/alpaqa/accelerators/internal/anderson-helpers.hpp

@@ -0,0 +1,92 @@
+#pragma once
+
+#include <alpaqa/accelerators/internal/limited-memory-qr.hpp>
+#include <alpaqa/config/config.hpp>
+#include <alpaqa/export.hpp>
+
+namespace alpaqa {
+
+/**
+ * @brief   Solve one step of Anderson acceleration to find a fixed point of a 
+ *          function g(x):
+ * 
+ * @f$ g(x^\star) - x^\star = 0 @f$
+ * 
+ * Updates the QR factorization of @f$ \mathcal{R}_k = QR @f$, solves the least 
+ * squares problem to find @f$ \gamma_\text{LS} @f$, computes the next 
+ * iterate @f$ x_{k+1} @f$, and stores the current function value @f$ g_k @f$
+ * in the matrix @f$ G @f$, which is used as a circular buffer.
+ * @f[ \begin{aligned}
+ * \def\gammaLS{\gamma_\text{LS}}
+ * m_k &= \min \{k, m\} \\
+ * g_i &= g(x_i) \\
+ * r_i &= r(x_i) g_i - x_i \\
+ * \Delta r_i &= r_i - r_{i-1} \\
+ * \mathcal{R}_k &= \begin{pmatrix} \Delta r_{k-m_k+1} & \dots & \Delta r_k \end{pmatrix} \in \R^{n\times m_k}
+ * \\
+ * \gammaLS &= \argmin_{\gamma \in \R^{m_k}}\; \norm{\mathcal{R}_k \gamma - r_k}_2 \\
+ * \alpha_i &= \begin{cases} \gammaLS[0] & i = 0 \\
+ * \gammaLS[i] - \gammaLS[i-1] & 0 \lt i \lt m_k \\
+ * 1 - \gammaLS[m_k - 1] & i = m_k \end{cases} \\
+ * \tilde G_k &= \begin{pmatrix} g_{k - m_k} & \dots & g_{k-1} \end{pmatrix} \\
+ * G_k &= \begin{pmatrix} g_{k - m_k} & \dots & g_{k} \end{pmatrix} \\
+ * &= \begin{pmatrix} \tilde G_k & g_{k} \end{pmatrix} \\
+ * x_{k+1} &= \sum_{i=0}^{m_k} \alpha_i\,g_{k - m_k + i} \\
+ * &= G_k \alpha \\
+ * \end{aligned} @f]
+ */
+template <Config Conf = DefaultConfig>
+inline void minimize_update_anderson(
+    /// [inout] QR factorization of @f$ \mathcal{R}_k @f$
+    LimitedMemoryQR<Conf> &qr,
+    /// [inout] Matrix of previous function values @f$ \tilde G_k @f$
+    ///         (stored as ring buffer with the same indices as `qr`)
+    rmat<Conf> G̃,
+    /// [in]    Current residual @f$ r_k @f$
+    crvec<Conf> rₖ,
+    /// [in]    Previous residual @f$ r_{k-1} @f$
+    crvec<Conf> rₗₐₛₜ,
+    /// [in]    Current function value @f$ g_k @f$
+    crvec<Conf> gₖ,
+    /// [in]    Minimum divisor when solving close to singular systems,
+    ///         scaled by the maximum eigenvalue of R
+    real_t<Conf> min_div_fac,
+    /// [out]   Solution to the least squares system
+    rvec<Conf> γ_LS,
+    /// [out]   Next Anderson iterate
+    rvec<Conf> xₖ_aa) {
+
+    // Update QR factorization for Anderson acceleration
+    if (qr.num_columns() == qr.m()) // if the history buffer is full
+        qr.remove_column();
+    qr.add_column(rₖ - rₗₐₛₜ);
+
+    // Solve least squares problem Anderson acceleration
+    // γ = argmin ‖ ΔR γ - rₖ ‖²
+    qr.solve_col(rₖ, γ_LS, qr.get_max_eig() * min_div_fac);
+
+    // Iterate over columns of G, whose indices match the indices of the matrix
+    // R in the QR factorization, stored as a circular buffer.
+    auto g_it  = qr.ring_iter().begin();
+    auto g_end = qr.ring_iter().end();
+    assert(g_it != g_end);
+
+    // Compute Anderson acceleration next iterate yₑₓₜ = ∑ₙ₌₀ αₙ gₙ
+    // α₀ = γ₀             if n = 0
+    // αₙ = γₙ - γₙ₋₁      if 0 < n < mₖ
+    // αₘ = 1 - γₘ₋₁       if n = mₖ
+    auto α = γ_LS(0);
+    xₖ_aa  = α * G̃.col((*g_it).circular);
+    while (++g_it != g_end) {
+        auto [i, g_idx] = *g_it; // [zero based index, circular index]
+        α               = γ_LS(i) - γ_LS(i - 1);
+        xₖ_aa += α * G̃.col(g_idx);
+    }
+    α = 1 - γ_LS(qr.num_columns() - 1);
+    xₖ_aa += α * gₖ;
+
+    // Add the new column to G
+    G̃.col(qr.ring_tail()) = gₖ; // TODO: avoid copy, make G an array of vectors
+}
+
+} // namespace alpaqa

casadi/include/alpaqa/accelerators/internal/limited-memory-qr.hpp

@@ -0,0 +1,295 @@
+#include <alpaqa/config/config.hpp>
+#include <alpaqa/util/ringbuffer.hpp>
+#include <Eigen/Jacobi>
+#include <type_traits>
+
+namespace alpaqa {
+
+/// Incremental QR factorization using modified Gram-Schmidt with
+/// reorthogonalization.
+///
+/// Computes A = QR while allowing efficient removal of the first
+/// column of A or adding new columns at the end of A.
+template <Config Conf = DefaultConfig>
+class LimitedMemoryQR {
+  public:
+    USING_ALPAQA_CONFIG(Conf);
+    LimitedMemoryQR() = default;
+
+    /// @param  n
+    ///         The size of the vectors, the number of rows of A.
+    /// @param  m
+    ///         The maximum number of columns of A.
+    ///
+    /// The maximum dimensions of Q are n×m and the maximum dimensions of R are
+    /// m×m.
+    LimitedMemoryQR(length_t n, length_t m) : Q(n, m), R(m, m) {}
+
+    length_t n() const { return Q.rows(); }
+    length_t m() const { return Q.cols(); }
+    length_t size() const { return n(); }
+    length_t history() const { return m(); }
+
+    /// Add the given column to the right.
+    template <class VecV>
+    void add_column(const VecV &v) {
+        assert(num_columns() < m());
+
+        auto q = Q.col(q_idx);
+        auto r = R.col(r_idx_end);
+
+        // Modified Gram-Schmidt to make q orthogonal to Q
+        q = v;
+        for (index_t i = 0; i < q_idx; ++i) {
+            real_t s = Q.col(i).dot(q);
+            r(i)     = s;
+            q -= s * Q.col(i);
+        }
+
+        // Compute the norms of orthogonalized q and original v
+        real_t norm_q = q.norm();
+        real_t norm_v = v.norm();
+
+        // If ‖q‖ is significantly smaller than ‖v‖, perform
+        // reorthogonalization
+        auto η = real_t(0.7);
+        while (norm_q < η * norm_v) {
+            ++reorth_count;
+            for (index_t i = 0; i < q_idx; ++i) {
+                real_t s = Q.col(i).dot(q);
+                r(i) += s;
+                q -= s * Q.col(i);
+            }
+            norm_v = norm_q;
+            norm_q = q.norm();
+        }
+
+        // Normalize q such that new matrix (Q q) remains orthogonal (i.e. has
+        // orthonormal columns)
+        r(q_idx) = norm_q;
+        q /= norm_q;
+        // Keep track of the minimum/maximum diagonal element of R
+        min_eig = std::min(min_eig, norm_q);
+        max_eig = std::max(max_eig, norm_q);
+
+        // Increment indices, add a column to Q and R.
+        ++q_idx;
+        r_idx_end = r_succ(r_idx_end);
+    }
+
+    /// Remove the leftmost column.
+    void remove_column() {
+        assert(num_columns() > 0);
+
+        // After removing the first column of the upper triangular matrix R,
+        // it becomes upper Hessenberg. Givens rotations are used to make it
+        // triangular again.
+        Eigen::JacobiRotation<real_t> G;
+        index_t r = 0;                   // row index of R
+        index_t c = r_succ(r_idx_start); // column index of R in storage
+        // Loop over the diagonal elements of R:
+        while (r < q_idx - 1) {
+            // Compute the Givens rotation that makes the subdiagonal element
+            // of column c of R zero.
+            G.makeGivens(R(r, c), R(r + 1, c), &R(r, c));
+            // Apply it to the remaining columns of R.
+            // Not the current column, because the diagonal element was updated
+            // by the makeGivens function, and the subdiagonal element doesn't
+            // have to be set to zero explicitly, it's implicit.
+            // Also not the previous columns, because they are already
+            // implicitly zero below the diagonal and this rotation wouldn't
+            // have any effect there.
+            // TODO: can this be sped up by applying it in two blocks instead
+            //       of column by column?
+            for (index_t cc = r_succ(c); cc != r_idx_end; cc = r_succ(cc))
+                R.col(cc).applyOnTheLeft(r, r + 1, G.adjoint());
+            // Apply the inverse of the Givens rotation to Q.
+            Q.block(0, 0, Q.rows(), q_idx).applyOnTheRight(r, r + 1, G);
+            // Keep track of the minimum/maximum diagonal element of R
+            min_eig = std::min(min_eig, R(r, c));
+            max_eig = std::max(max_eig, R(r, c));
+            // Advance indices to next diagonal element of R.
+            ++r;
+            c = r_succ(c);
+        }
+        // Remove rightmost column of Q, since it corresponds to the bottom row
+        // of R, which was set to zero by the Givens rotations
+        --q_idx;
+        // Remove the first column of R.
+        r_idx_start = r_succ(r_idx_start);
+    }
+
+    /// Solve the least squares problem Ax = b.
+    /// Do not divide by elements that are smaller in absolute value than @p tol.
+    template <class VecB, class VecX>
+    void solve_col(const VecB &b, VecX &x, real_t tol = 0) const {
+        // Iterate over the diagonal of R, starting at the bottom right,
+        // this is standard back substitution
+        // (recall that R is stored in a circular buffer, so R.col(i) is
+        // not the mathematical i-th column)
+        auto rev_bgn = ring_reverse_iter().begin();
+        auto rev_end = ring_reverse_iter().end();
+        auto fwd_end = ring_iter().end();
+        for (auto it_d = rev_bgn; it_d != rev_end; ++it_d) {
+            // Row/column index of diagonal element of R
+            auto [rR, cR] = *it_d;
+            // Don't divide by very small diagonal elements
+            if (std::abs(R(rR, cR)) < tol) {
+                x(rR) = real_t{0};
+                continue;
+            }
+            // (r is the zero-based mathematical index, c is the index in
+            // the circular buffer)
+            x(rR) = Q.col(rR).transpose() * b; // Compute rhs Qᵀb
+            // In the current row of R, iterate over the elements to the
+            // right of the diagonal
+            // Iterating from left to right seems to give better results
+            for (auto it_c = it_d.forwardit; it_c != fwd_end; ++it_c) {
+                auto [rX2, cR2] = *it_c;
+                x(rR) -= R(rR, cR2) * x(rX2);
+            }
+            x(rR) /= R(rR, cR); // Divide by diagonal element
+        }
+    }
+
+    /// Solve the least squares problem AX = B.
+    /// Do not divide by elements that are smaller in absolute value than @p tol.
+    template <class MatB, class MatX>
+    void solve(const MatB &B, MatX &X, real_t tol = 0) const {
+        assert(B.cols() <= X.cols());
+        assert(B.rows() >= Q.rows());
+        assert(X.rows() >= Eigen::Index(num_columns()));
+        // Each column of the right hand side is solved as an individual system
+        for (Eigen::Index cB = 0; cB < B.cols(); ++cB) {
+            auto b = B.col(cB);
+            auto x = X.col(cB);
+            solve_col(b, x, tol);
+        }
+    }
+
+    template <class Derived>
+    using solve_ret_t = std::conditional_t<
+        Eigen::internal::traits<Derived>::ColsAtCompileTime == 1, vec, mat>;
+
+    /// Solve the least squares problem AX = B.
+    template <class Derived>
+    solve_ret_t<Derived> solve(const Eigen::DenseBase<Derived> &B) {
+        solve_ret_t<Derived> X(m(), B.cols());
+        solve(B, X);
+        return X;
+    }
+
+    /// Get the full, raw storage for the orthogonal matrix Q.
+    const mat &get_raw_Q() const { return Q; }
+    /// Get the full, raw storage for the upper triangular matrix R.
+    /// The columns of this matrix are permuted because it's stored as a
+    /// circular buffer for efficiently appending columns to the end and
+    /// popping columns from the front.
+    const mat &get_raw_R() const { return R; }
+
+    /// Get the full storage for the upper triangular matrix R but with the
+    /// columns in the correct order.
+    /// @note   Meant for tests only, creates a permuted copy.
+    mat get_full_R() const {
+        if (r_idx_start == 0)
+            return R;
+        // Using a permutation matrix here isn't as efficient as rotating the
+        // matrix manually, but this function is only used in tests, so it
+        // shouldn't matter.
+        Eigen::PermutationMatrix<Eigen::Dynamic> P(R.cols());
+        P.setIdentity();
+        std::rotate(P.indices().data(), P.indices().data() + r_idx_start,
+                    P.indices().data() + P.size());
+        return R * P;
+    }
+    /// Get the matrix R such that Q times R is the original matrix.
+    /// @note   Meant for tests only, creates a permuted copy.
+    mat get_R() const {
+        return get_full_R()
+            .block(0, 0, q_idx, q_idx)
+            .template triangularView<Eigen::Upper>();
+    }
+    /// Get the matrix Q such that Q times R is the original matrix.
+    /// @note   Meant for tests only, creates a copy.
+    mat get_Q() const { return Q.block(0, 0, n(), q_idx); }
+
+    /// Multiply the matrix R by a scalar.
+    void scale_R(real_t scal) {
+        for (auto [i, r_idx] : ring_iter())
+            R.col(r_idx).topRows(i + 1) *= scal;
+        min_eig *= scal;
+        max_eig *= scal;
+    }
+
+    /// Get the number of MGS reorthogonalizations.
+    unsigned long get_reorth_count() const { return reorth_count; }
+    /// Reset the number of MGS reorthogonalizations.
+    void clear_reorth_count() { reorth_count = 0; }
+
+    /// Get the minimum eigenvalue of R.
+    real_t get_min_eig() const { return min_eig; }
+    /// Get the maximum eigenvalue of R.
+    real_t get_max_eig() const { return max_eig; }
+
+    /// Reset all indices, clearing the Q and R matrices.
+    void reset() {
+        q_idx        = 0;
+        r_idx_start  = 0;
+        r_idx_end    = 0;
+        reorth_count = 0;
+        min_eig      = +inf<config_t>;
+        max_eig      = -inf<config_t>;
+    }
+
+    /// Re-allocate storage for a problem with a different size. Causes
+    /// a @ref reset.
+    void resize(length_t n, length_t m) {
+        Q.resize(n, m);
+        R.resize(m, m);
+        reset();
+    }
+
+    /// Get the number of columns that are currently stored.
+    length_t num_columns() const { return q_idx; }
+    /// Get the head index of the circular buffer (points to the oldest
+    /// element).
+    index_t ring_head() const { return r_idx_start; }
+    /// Get the tail index of the circular buffer (points to one past the most
+    /// recent element).
+    index_t ring_tail() const { return r_idx_end; }
+    /// Get the next index in the circular buffer.
+    index_t ring_next(index_t i) const { return r_succ(i); }
+    /// Get the previous index in the circular buffer.
+    index_t ring_prev(index_t i) const { return r_pred(i); }
+    /// Get the number of columns currently stored in the buffer.
+    length_t current_history() const { return q_idx; }
+
+    /// Get iterators in the circular buffer.
+    CircularRange<index_t> ring_iter() const {
+        return {q_idx, r_idx_start, r_idx_end, m()};
+    }
+    /// Get reverse iterators in the circular buffer.
+    ReverseCircularRange<index_t> ring_reverse_iter() const {
+        return ring_iter();
+    }
+
+  private:
+    mat Q; ///< Storage for orthogonal factor Q.
+    mat R; ///< Storage for upper triangular factor R.
+
+    index_t q_idx       = 0; ///< Number of columns of Q being stored.
+    index_t r_idx_start = 0; ///< Index of the first column of R.
+    index_t r_idx_end   = 0; ///< Index of the one-past-last column of R.
+
+    unsigned long reorth_count = 0; ///< Number of MGS reorthogonalizations.
+
+    real_t min_eig = +inf<config_t>; ///< Minimum eigenvalue of R.
+    real_t max_eig = -inf<config_t>; ///< Maximum eigenvalue of R.
+
+    /// Get the next index in the circular storage for R.
+    index_t r_succ(index_t i) const { return i + 1 < m() ? i + 1 : 0; }
+    /// Get the previous index in the circular storage for R.
+    index_t r_pred(index_t i) const { return i == 0 ? m() - 1 : i - 1; }
+};
+
+} // namespace alpaqa