gpboost 1.3.0__tar.gz → 1.3.3__tar.gz

{gpboost-1.3.0/gpboost.egg-info → gpboost-1.3.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: gpboost
-Version: 1.3.0
+Version: 1.3.3
 Summary: GPBoost Python Package
 Home-page: https://github.com/fabsig/GPBoost
 Maintainer: Fabio Sigrist

{gpboost-1.3.0 → gpboost-1.3.3}/compile/external_libs/LBFGSpp/include/LBFGS.h

@@ -1,5 +1,5 @@
 // Copyright (C) 2016-2023 Yixuan Qiu <yixuan.qiu@cos.name>
-//  Modified work Copyright (c) 2024 Fabio Sigrist. All rights reserved.
+// Modified work Copyright (c) 2024 Fabio Sigrist. All rights reserved.
 // Under MIT license
 
 #ifndef LBFGSPP_LBFGS_H
@@ -89,7 +89,30 @@ public:
         const int fpast = m_param.past;
 
         // Evaluate function and compute gradient
-        fx = f(x, m_grad);
+        fx = f(x, m_grad, true, true);// ChangedForGPBoost
+
+        std::string init_coef_str = "";
+        if (f.HasCovariates())
+        {
+            init_coef_str = " and 'init_coef'";
+        }
+        std::string problem_str = "none";
+        if (std::isnan(fx))
+        {
+            problem_str = "NaN";
+        }
+        else if (std::isinf(fx))
+        {
+            problem_str = "Inf";
+        }
+        if (problem_str != "none")
+        {
+            Log::REFatal((problem_str + " occurred in initial approximate negative marginal log-likelihood. "
+                "Possible solutions: try other initial values ('init_cov_pars'" + init_coef_str + ") "
+                "or other tuning parameters in case you apply the GPBoost algorithm (e.g., learning_rate)").c_str());
+        }
+        Log::REDebug("Initial approximate negative marginal log-likelihood: %g", fx);
+
         m_gnorm = m_grad.norm();
         if (fpast > 0)
             m_fx[0] = fx;
@@ -152,7 +175,9 @@ public:
             if (fpast > 0)
             {
                 const Scalar fxd = m_fx[k % fpast];
-                if (k >= fpast && abs(fxd - fx) <= m_param.delta * std::max(std::max(abs(fx), abs(fxd)), Scalar(1)))
+
+                // ChangedForGPBoost
+                if (k >= fpast && (fxd - fx) <= m_param.delta * std::max(abs(fxd), Scalar(1)))
                     return k;
 
                 m_fx[k % fpast] = fx;
@@ -182,7 +207,7 @@ public:
             if (f.LearnCovarianceParameters() && f.ShouldRedetermineNearestNeighborsVecchia())
             {
                 f.RedetermineNearestNeighborsVecchia();  // called only in certain iterations if gp_approx == "vecchia" and neighbors are selected based on correlations and not distances
-                fx = f(x, m_grad, false);
+                fx = f(x, m_grad, true, false);
                 m_bfgs.apply_Hv(m_grad, -Scalar(1), m_drt);
                 if (fpast > 0)
                 {

{gpboost-1.3.0 → gpboost-1.3.3}/compile/external_libs/LBFGSpp/include/LBFGSpp/LineSearchBacktracking.h

@@ -1,4 +1,5 @@
 // Copyright (C) 2016-2023 Yixuan Qiu <yixuan.qiu@cos.name>
+// Modified work Copyright (c) 2024 Fabio Sigrist. All rights reserved.
 // Under MIT license
 
 #ifndef LBFGSPP_LINE_SEARCH_BACKTRACKING_H
@@ -70,11 +71,21 @@ public:
             // x_{k+1} = x_k + step * d_k
             x.noalias() = xp + step * drt;
             // Evaluate this candidate
-            fx = f(x, grad);
+            fx = f(x, grad, true, false);  // ChangedForGPBoost
+
+            //Log::REInfo("LineSearch: iter = %d, fx = %g, step = %g, fx_init = %g", iter, fx, step, fx_init);  // for debugging
 
             if (fx > fx_init + step * test_decr || (fx != fx))
             {
-                width = dec;
+                // ChangedForGPBoost
+                if ((fx - fx_init) > 2. * std::max(abs(fx_init), Scalar(1)))
+                {
+                    width = dec / 16.;  // make step size much smaller for very large increases to avoid too many backtracking steps
+                }
+                else
+                {
+                    width = dec;
+                }
             }
             else
             {
@@ -115,8 +126,20 @@ public:
             step *= width;
         }
 
+        // ChangedForGPBoost
         if (iter >= param.max_linesearch)
+        {
+            x.noalias() = xp;
+            fx = fx_init;
+            step = 0.;
             Log::REDebug("GPModel lbfgs: the line search routine reached the maximum number of iterations");
+        }
+        else if (iter > 0)
+        {
+            Log::REDebug("LineSearch for 'lbfgs' finished after %d iterations, step length = %g", iter, step);
+        }
+        f(x, grad, false, true);//calculate gradient
+         
     }
 };
 

{gpboost-1.3.0 → gpboost-1.3.3}/compile/external_libs/LBFGSpp/include/LBFGSpp/LineSearchNocedalWright.h

@@ -1,5 +1,6 @@
 // Copyright (C) 2016-2023 Yixuan Qiu <yixuan.qiu@cos.name>
 // Copyright (C) 2016-2023 Dirk Toewe <DirkToewe@GoogleMail.com>
+// Modified work Copyright (c) 2024 Fabio Sigrist. All rights reserved.
 // Under MIT license
 
 #ifndef LBFGSPP_LINE_SEARCH_NOCEDAL_WRIGHT_H
@@ -141,7 +142,7 @@ public:
         {
             // Evaluate the current step size
             x.noalias() = xp + step * drt;
-            fx = f(x, grad);
+            fx = f(x, grad, true, false);
             dg = grad.dot(drt);
 
             // Test the sufficient decrease condition
@@ -157,7 +158,11 @@ public:
 
             // Test the curvature condition
             if (std::abs(dg) <= test_curv)
+            {
+                f(x, grad, false, true);//calculate gradient
                 return;  // Case (4)
+            }
+                
 
             step_hi = step_lo;
             fx_hi = fx_lo;
@@ -187,6 +192,7 @@ public:
                 // But we need to move {x, grad}_lo back before returning
                 x.swap(x_lo);
                 grad.swap(grad_lo);
+                f(x, grad, false, true);//calculate gradient
                 return;
             }
 
@@ -214,7 +220,7 @@ public:
 
             // Evaluate the current step size
             x.noalias() = xp + step * drt;
-            fx = f(x, grad);
+            fx = f(x, grad, true, false);
             dg = grad.dot(drt);
 
             // Test the sufficient decrease condition
@@ -231,6 +237,7 @@ public:
             {
                 // Test the curvature condition
                 if (std::abs(dg) <= test_curv)
+                    f(x, grad, false, true);//calculate gradient
                     return;
 
                 if (dg * (step_hi - step_lo) >= Scalar(0))
@@ -261,15 +268,23 @@ public:
             {
                 // throw std::runtime_error("the line search routine reached the maximum number of iterations");
                 if (step_lo <= Scalar(0))
-                    Log::REFatal("GPModel lbfgs: the line search routine failed, unable to sufficiently decrease the function value");
-
-                // Return everything with _lo
-                step = step_lo;
-                fx = fx_lo;
-                dg = dg_lo;
-                // Move {x, grad}_lo back
-                x.swap(x_lo);
-                grad.swap(grad_lo);
+                {
+                    x.noalias() = xp;
+                    fx = fx_init;
+                    step = 0.;
+                    Log::REDebug("GPModel lbfgs: the line search routine reached the maximum number of iterations");
+                }
+                else
+                {
+                    // Return everything with _lo
+                    step = step_lo;
+                    fx = fx_lo;
+                    dg = dg_lo;
+                    // Move {x, grad}_lo back
+                    x.swap(x_lo);
+                    grad.swap(grad_lo);
+                }
+                f(x, grad, false, true);//calculate gradient
                 return;
             }
         }

{gpboost-1.3.0 → gpboost-1.3.3}/compile/include/GPBoost/CG_utils.h

@@ -22,7 +22,8 @@ namespace GPBoost {
 	/*!
 	* \brief Preconditioned conjugate gradient descent to solve A u = rhs when rhs is a vector
 	*		 A = (Sigma^-1 + W) is a symmetric matrix of dimension nxn, a Vecchia approximation for Sigma^-1,
-	*		 Sigma^-1 = B^T D^-1 B, is given, and W is a diagonal matrix. P = B^T (D^-1 + W) B is used as preconditioner.
+	*		 Sigma^-1 = B^T D^-1 B, is given, and W is a diagonal matrix.
+	*		 "Sigma_inv_plus_BtWB" (P = B^T (D^-1 + W) B)  or "zero_infill_incomplete_cholesky" (P = L^T L) is used as preconditioner.
 	* \param diag_W Diagonal of matrix W
 	* \param B_rm Row-major matrix B in Vecchia approximation Sigma^-1 = B^T D^(-1) B ("=" Cholesky factor)
 	* \param B_t_D_inv_rm Row-major matrix that contains the product B^T D^-1. Outsourced in order to reduce the overhead of the function.
@@ -33,7 +34,9 @@ namespace GPBoost {
 	* \param find_mode_it In the first mode-finding iteration (find_mode_it == 0) u is set to zero at the beginning of the algorithm (cold-start).
 	* \param delta_conv Tolerance for checking convergence of the algorithm
 	* \param THRESHOLD_ZERO_RHS_CG If the L1-norm of the rhs is below this threshold the CG is not executed and a vector u of 0's is returned.
+	* \param cg_preconditioner_type Type of preconditioner used.
 	* \param D_inv_plus_W_B_rm Row-major matrix that contains the product (D^(-1) + W) B used for the preconditioner "Sigma_inv_plus_BtWB".
+	* \param L_SigmaI_plus_W_rm Row-major matrix that contains sparse cholesky factor L of matrix L^T L =  B^T D^(-1) B + W used for the preconditioner "zero_infill_incomplete_cholesky". 
 	*/
 	void CGVecchiaLaplaceVec(const vec_t& diag_W,
 		const sp_mat_rm_t& B_rm,
@@ -45,7 +48,9 @@ namespace GPBoost {
 		const int find_mode_it,
 		const double delta_conv,
 		const double THRESHOLD_ZERO_RHS_CG,
-		const sp_mat_rm_t& D_inv_plus_W_B_rm);
+		const string_t cg_preconditioner_type,
+		const sp_mat_rm_t& D_inv_plus_W_B_rm,
+		const sp_mat_rm_t& L_SigmaI_plus_W_rm);
 
 	/*!
 	* \brief Version of CGVecchiaLaplaceVec() that solves (Sigma^-1 + W) u = rhs by u = W^(-1) (W^(-1) + Sigma)^(-1) Sigma rhs where the preconditioned conjugate 
@@ -81,7 +86,8 @@ namespace GPBoost {
 	/*!
 	* \brief Preconditioned conjugate gradient descent in combination with the Lanczos algorithm.
 	*		 A linear system A U = rhs is solved, where the rhs is a matrix of dimension nxt of t random column-vectors and 
-	*		 A = (Sigma^-1 + W) is a symmetric matrix of dimension nxn. P = B^T (D^-1 + W) B is used as preconditioner.
+	*		 A = (Sigma^-1 + W) is a symmetric matrix of dimension nxn. 
+	*		 "Sigma_inv_plus_BtWB" (P = B^T (D^-1 + W) B)  or "zero_infill_incomplete_cholesky" (P = L^T L) is used as preconditioner.
 	*		 Further, a Vecchia approximation for Sigma^-1 = B^T D^-1 B is given, and W is a diagonal matrix.
 	*		 The function returns t approximative tridiagonalizations T of the symmetric matrix P^(-0.5) A P^(-0.5) = Q T Q^T in vector form (diagonal + subdiagonal of T)
 	*		 and an approximative solution of the linear system.
@@ -97,7 +103,9 @@ namespace GPBoost {
 	* \param t t-Dimension of the linear system
 	* \param p Maximal number of conjugate gradient steps
 	* \param delta_conv Tolerance for checking convergence of the algorithm
+	* \param cg_preconditioner_type Type of preconditioner used.
 	* \param D_inv_plus_W_B_rm Row-major matrix that contains the product (D^(-1) + W) B used for the preconditioner "Sigma_inv_plus_BtWB".
+	* \param L_SigmaI_plus_W_rm Row-major matrix that contains sparse cholesky factor L of matrix L^T L =  B^T D^(-1) B + W used for the preconditioner "zero_infill_incomplete_cholesky".
 	*/
 	void CGTridiagVecchiaLaplace(const vec_t& diag_W,
 		const sp_mat_rm_t& B_rm,
@@ -111,7 +119,9 @@ namespace GPBoost {
 		const int t,
 		int p,
 		const double delta_conv,
-		const sp_mat_rm_t& D_inv_plus_W_B_rm);
+		const string_t cg_preconditioner_type,
+		const sp_mat_rm_t& D_inv_plus_W_B_rm,
+		const sp_mat_rm_t& L_SigmaI_plus_W_rm);
 
 	/*!
 	* \brief Version of CGTridiagVecchiaLaplace() where A = (W^(-1) + Sigma).
@@ -208,6 +218,16 @@ namespace GPBoost {
 		const vec_t& tr_BI_B_deriv,
 		vec_t& c_opt);
 
+	/*!
+	* \brief Reverse incomplete Cholesky factorization L^T L = A under the constrain that L has the same sparcity pattern as A or B.
+	* \param A Column-major matrix to factorize.
+	* \param B Column-major matrix providing the sparcity pattern.
+	* \param L_rm[out] Row-major matrix containing the sparse lower triangular factor L. 
+	*/
+	void ReverseIncompleteCholeskyFactorization(sp_mat_t& A,
+		const sp_mat_t& B, 
+		sp_mat_rm_t& L_rm);
+
 	/*!
 	* \brief Pivoted Cholesky factorization according to Habrecht et al. (2012) for the original (nonapproximated) covariance matrix (Sigma)
 	* \param cov_f Pointer to function which accesses elements of Sigma (see https://www.geeksforgeeks.org/passing-a-function-as-a-parameter-in-cpp)

{gpboost-1.3.0 → gpboost-1.3.3}/compile/include/GPBoost/GP_utils.h

@@ -106,6 +106,7 @@ namespace GPBoost {
 		}
 		dist = T_mat(coords2.rows(), coords1.rows());
 		dist.setFromTriplets(triplets.begin(), triplets.end());
+		dist.makeCompressed();
 	}//end CalculateDistances (sparse)
 
 	/*!

{gpboost-1.3.0 → gpboost-1.3.3}/compile/include/GPBoost/Vecchia_utils.h

@@ -2,7 +2,7 @@
 * This file is part of GPBoost a C++ library for combining
 *	boosting with Gaussian process and mixed effects models
 *
-* Copyright (c) 2020 Fabio Sigrist. All rights reserved.
+* Copyright (c) 2020 - 2024 Fabio Sigrist. All rights reserved.
 *
 * Licensed under the Apache License Version 2.0. See LICENSE file in the project root for license information.
 */
@@ -218,7 +218,8 @@ namespace GPBoost {
 					cov_fct_taper_shape,
 					re_comp->GetTaperMu(),
 					apply_tapering,
-					false)));
+					false,
+					dim_gp_coords)));
 				//save random coefficient data in the form ot outer product matrices
 #pragma omp for schedule(static)
 				for (int i = 0; i < num_data_per_cluster[cluster_i]; ++i) {
@@ -263,12 +264,9 @@ namespace GPBoost {
 		CHECK(re_comp->ShouldSaveDistances() == false);
 		int num_re = re_comp->GetNumUniqueREs();
 		CHECK((int)nearest_neighbors_cluster_i.size() == num_re);
-		// calculate scale coordinates
-		int dim_space = re_comp->GetDimSpace();
-		const vec_t pars = re_comp->CovPars();
-		den_mat_t coords_scaled(num_re, re_comp->GetDimCoords());
-		coords_scaled.col(0) = (re_comp->GetCoords()).col(0) * pars[1];
-		coords_scaled.rightCols(dim_space) = (re_comp->GetCoords()).rightCols(dim_space) * pars[2];
+		// Calculate scaled coordinates
+		den_mat_t coords_scaled;
+		re_comp->GetScaledCoordinates(coords_scaled);
 		// find correlation-based nearest neighbors
 		std::vector<den_mat_t> dist_dummy;
 		bool check_has_duplicates = false;
@@ -584,11 +582,8 @@ namespace GPBoost {
 		bool distances_saved = re_comp->ShouldSaveDistances();
 		den_mat_t coords_scaled;
 		if (!distances_saved) {
-			int dim_space = re_comp->GetDimSpace();
 			const vec_t pars = re_comp->CovPars();
-			coords_scaled = den_mat_t(coords_all.rows(), coords_all.cols());
-			coords_scaled.col(0) = coords_all.col(0) * pars[1];
-			coords_scaled.rightCols(dim_space) = coords_all.rightCols(dim_space) * pars[2];
+			re_comp->ScaleCoordinates(pars, coords_all, coords_scaled);
 		}
 		if (CondObsOnly) {
 			if (distances_saved) {
@@ -832,11 +827,8 @@ namespace GPBoost {
 				vecchia_neighbor_selection, rng, distances_saved);
 		}
 		else {
-			int dim_space = re_comp->GetDimSpace();
 			const vec_t pars = re_comp->CovPars();
-			coords_scaled = den_mat_t(coords_all.rows(), coords_all.cols());
-			coords_scaled.col(0) = coords_all.col(0) * pars[1];
-			coords_scaled.rightCols(dim_space) = coords_all.rightCols(dim_space) * pars[2];
+			re_comp->ScaleCoordinates(pars, coords_all, coords_scaled);
 			find_nearest_neighbors_Vecchia_fast(coords_scaled, num_data_tot, num_neighbors_pred,
 				nearest_neighbors_cluster_i, dist_obs_neighbors_cluster_i, dist_between_neighbors_cluster_i, 0, -1, check_has_duplicates,
 				vecchia_neighbor_selection, rng, distances_saved);
@@ -1084,11 +1076,8 @@ namespace GPBoost {
 		bool distances_saved = re_comp->ShouldSaveDistances();
 		den_mat_t coords_scaled;
 		if (!distances_saved) {
-			int dim_space = re_comp->GetDimSpace();
 			const vec_t pars = re_comp->CovPars();
-			coords_scaled = den_mat_t(coords_all_unique.rows(), coords_all_unique.cols());
-			coords_scaled.col(0) = coords_all_unique.col(0) * pars[1];
-			coords_scaled.rightCols(dim_space) = coords_all_unique.rightCols(dim_space) * pars[2];
+			re_comp->ScaleCoordinates(pars, coords_all_unique, coords_scaled);
 		}
 		if (CondObsOnly) {//find neighbors among both the observed locations only
 			if (distances_saved) {