CUQIpy 1.3.0.post0.dev401__py3-none-any.whl → 1.4.0.post0.dev41__py3-none-any.whl

cuqi/__init__.py

@@ -11,6 +11,7 @@ from . import operator
 from . import pde
 from . import problem
 from . import sampler
+from . import legacy
 from . import array
 from . import samples
 from . import solver

cuqi/_version.py

@@ -8,11 +8,11 @@ import json
 
 version_json = '''
 {
- "date": "2025-09-21T15:00:42+0200",
+ "date": "2025-10-09T13:25:50+0200",
  "dirty": false,
  "error": null,
- "full-revisionid": "f08eb2ae4f6f5209d4a06a4e031bc61bb746ce01",
- "version": "1.3.0.post0.dev401"
+ "full-revisionid": "92bb2e16f3828d8074c008d4ed6a08edcae0889d",
+ "version": "1.4.0.post0.dev41"
 }
 '''  # END VERSION_JSON
 

cuqi/density/_density.py

@@ -143,7 +143,15 @@ class Density(ABC):
     def enable_FD(self, epsilon=1e-8):
         """ Enable finite difference approximation for logd gradient. Note
         that if enabled, the FD approximation will be used even if the 
-        _gradient method is implemented. """
+        _gradient method is implemented. 
+        
+        Parameters
+        ----------
+        epsilon : float
+
+        Spacing (step size) to use for finite difference approximation for logd
+        gradient for each variable. Default is 1e-8.
+        """
         self._FD_enabled = True
         self._FD_epsilon = epsilon
 

cuqi/distribution/_joint_distribution.py

@@ -84,6 +84,8 @@ class JointDistribution:
         cond_vars = self._get_conditioning_variables()
         if len(cond_vars) > 0:
             raise ValueError(f"Every density parameter must have a distribution (prior). Missing prior for {cond_vars}.")
+        # Initialize finite difference gradient approximation settings
+        self.disable_FD()
 
     # --------- Public properties ---------
     @property
@@ -96,6 +98,38 @@ class JointDistribution:
         """ Returns the geometries of the joint distribution. """
         return [dist.geometry for dist in self._distributions]
 
+    @property
+    def FD_enabled(self):
+        """ Returns a dictionary of keys and booleans indicating for each
+        parameter name (key) if finite difference approximation of the logd
+        gradient is enabled. """
+        par_names = self.get_parameter_names()
+        FD_enabled = {
+            par_name: self.FD_epsilon[par_name] is not None for par_name in par_names
+        }
+        return FD_enabled
+
+    @property
+    def FD_epsilon(self):
+        """ Returns a dictionary indicating for each parameter name the
+        spacing for the finite difference approximation of the logd gradient."""
+        return self._FD_epsilon
+
+    @FD_epsilon.setter
+    def FD_epsilon(self, value):
+        """ Set the spacing for the finite difference approximation of the
+        logd gradient as a dictionary. The keys are the parameter names.
+        The value for each key is either None (no FD approximation) or a float
+        representing the FD step size.
+        """
+        par_names = self.get_parameter_names()
+        if value is None:
+            self._FD_epsilon = {par_name: None for par_name in par_names}
+        else:
+            if set(value.keys()) != set(par_names):
+                raise ValueError("Keys of FD_epsilon must match the parameter names of the distribution "+f" {par_names}")
+            self._FD_epsilon = value
+
     # --------- Public methods ---------
     def logd(self, *args, **kwargs):
         """ Evaluate the un-normalized log density function. """
@@ -136,6 +170,33 @@ class JointDistribution:
         # Can reduce to Posterior, Likelihood or Distribution.
         return new_joint._reduce_to_single_density()
 
+    def enable_FD(self, epsilon=None):
+        """ Enable finite difference approximation for logd gradient. Note
+        that if enabled, the FD approximation will be used even if the 
+        _gradient method is implemented. By default, all parameters
+        will have FD enabled with a step size of 1e-8.
+        
+        Parameters
+        ----------
+        epsilon : dict, *optional*
+
+        Dictionary indicating the spacing (step size) to use for finite
+        difference approximation for logd gradient for each variable.
+
+        Keys are variable names.
+        Values are either a float to enable FD with the given value as the FD
+        step size, or None to disable FD for that variable. Default is 1e-8 for
+        all variables.
+        """
+        if epsilon is None:
+            epsilon = {par_name: 1e-8 for par_name in self.get_parameter_names()}
+        self.FD_epsilon = epsilon
+
+    def disable_FD(self):
+        """ Disable finite difference approximation for logd gradient. """
+        par_names = self.get_parameter_names()
+        self.FD_epsilon = {par_name: None for par_name in par_names}
+
     def get_parameter_names(self) -> List[str]:
         """ Returns the parameter names of the joint distribution. """
         return [dist.name for dist in self._distributions]
@@ -202,34 +263,58 @@ class JointDistribution:
         # Count number of distributions and likelihoods
         n_dist = len(self._distributions)
         n_likelihood = len(self._likelihoods)
+        reduced_FD_epsilon = {par_name:self.FD_epsilon[par_name] for par_name in self.get_parameter_names()}
+        self.enable_FD(epsilon=reduced_FD_epsilon)
 
         # Cant reduce if there are multiple distributions or likelihoods
         if n_dist > 1:
             return self
 
+        # If only evaluated densities left return joint to ensure logd method is available
+        if n_dist == 0 and n_likelihood == 0:
+            return self
+
+        # Extract the parameter name of the distribution
+        if n_dist == 1:
+            par_name = self._distributions[0].name
+        elif n_likelihood == 1:
+            par_name = self._likelihoods[0].name
+        else:
+            par_name = None
+
         # If exactly one distribution and multiple likelihoods reduce
         if n_dist == 1 and n_likelihood > 1:
-            return MultipleLikelihoodPosterior(*self._densities)
-        
+            reduced_distribution = MultipleLikelihoodPosterior(*self._densities)
+            reduced_FD_epsilon = {par_name:self.FD_epsilon[par_name]}
+
         # If exactly one distribution and one likelihood its a Posterior
         if n_dist == 1 and n_likelihood == 1:
             # Ensure parameter names match, otherwise return the joint distribution
             if set(self._likelihoods[0].get_parameter_names()) != set(self._distributions[0].get_parameter_names()):
                 return self
-            return self._add_constants_to_density(Posterior(self._likelihoods[0], self._distributions[0]))
+            reduced_distribution = Posterior(self._likelihoods[0], self._distributions[0])
+            reduced_distribution = self._add_constants_to_density(reduced_distribution)
+            reduced_FD_epsilon = self.FD_epsilon[par_name]
 
         # If exactly one distribution and no likelihoods its a Distribution
         if n_dist == 1 and n_likelihood == 0:
-            return self._add_constants_to_density(self._distributions[0])        
-        
+            # Intentionally skip enabling FD here. If the user wants FD, they
+            # can enable it for this particular distribution before forming
+            # the joint distribution.
+            return self._add_constants_to_density(self._distributions[0])
+
         # If no distributions and exactly one likelihood its a Likelihood
         if n_likelihood == 1 and n_dist == 0:
-            return self._likelihoods[0]
+            # This case seems to not happen in practice, but we include it for
+            # completeness.
+            reduced_distribution = self._likelihoods[0]
+            reduced_FD_epsilon = self.FD_epsilon[par_name] 
+
+        if self.FD_enabled[par_name]:
+            reduced_distribution.enable_FD(epsilon=reduced_FD_epsilon)
+
+        return reduced_distribution
 
-        # If only evaluated densities left return joint to ensure logd method is available
-        if n_dist == 0 and n_likelihood == 0:
-            return self
-        
     def _add_constants_to_density(self, density: Density):
         """ Add the constants (evaluated densities) to a single density. Used when reducing to single density. """
 
@@ -274,7 +359,7 @@ class JointDistribution:
                     if len(cond_vars) > 0:
                         msg += f"|{cond_vars}"
                     msg += ")"
-        
+
         msg += "\n"
         msg += "    Densities: \n"
 

cuqi/experimental/__init__.py

@@ -1,5 +1,4 @@
 """ Experimental module for testing new features and ideas.  """
-from . import mcmc
 from . import algebra
 from . import geometry
-from ._recommender import SamplerRecommender
+from ._recommender import SamplerRecommender

cuqi/experimental/_recommender.py

@@ -3,7 +3,7 @@ import inspect
 import numpy as np
 
 # This import makes suggest_sampler easier to read
-import cuqi.experimental.mcmc as samplers 
+import cuqi.sampler as samplers 
 
 
 class SamplerRecommender(object):
@@ -15,7 +15,7 @@ class SamplerRecommender(object):
     target: Density or JointDistribution
         Distribution to get sampler recommendations for.
 
-    exceptions: list[cuqi.experimental.mcmc.Sampler], *optional*
+    exceptions: list[cuqi.sampler.Sampler], *optional*
         Samplers not to be recommended.
     
     Example
@@ -104,7 +104,7 @@ class SamplerRecommender(object):
 
         """
 
-        all_samplers = [(name, cls) for name, cls in inspect.getmembers(cuqi.experimental.mcmc, inspect.isclass) if issubclass(cls, cuqi.experimental.mcmc.Sampler)]
+        all_samplers = [(name, cls) for name, cls in inspect.getmembers(cuqi.sampler, inspect.isclass) if issubclass(cls, cuqi.sampler.Sampler)]
         valid_samplers = []
 
         for name, sampler in all_samplers:
@@ -116,7 +116,7 @@ class SamplerRecommender(object):
 
         # Need a separate case for HybridGibbs
         if self.valid_HybridGibbs_sampling_strategy() is not None:
-            valid_samplers += [cuqi.experimental.mcmc.HybridGibbs.__name__ if as_string else cuqi.experimental.mcmc.HybridGibbs]
+            valid_samplers += [cuqi.sampler.HybridGibbs.__name__ if as_string else cuqi.sampler.HybridGibbs]
 
         return valid_samplers
 

cuqi/legacy/__init__.py

@@ -0,0 +1,2 @@
+""" Legacy module for functionalities that are no longer supported or developed. """
+from . import sampler

cuqi/legacy/sampler/__init__.py

@@ -0,0 +1,11 @@
+from ._sampler import Sampler, ProposalBasedSampler
+from ._conjugate import Conjugate
+from ._conjugate_approx import ConjugateApprox
+from ._cwmh import CWMH
+from ._gibbs import Gibbs
+from ._hmc import NUTS
+from ._langevin_algorithm import ULA, MALA
+from ._laplace_approximation import UGLA
+from ._mh import MH
+from ._pcn import pCN
+from ._rto import LinearRTO, RegularizedLinearRTO

cuqi/legacy/sampler/_conjugate.py

@@ -0,0 +1,55 @@
+from cuqi.distribution import Posterior, Gaussian, Gamma, GMRF
+from cuqi.implicitprior import RegularizedGaussian, RegularizedGMRF
+import numpy as np
+
+class Conjugate: # TODO: Subclass from Sampler once updated
+    """ Conjugate sampler
+
+    Sampler for sampling a posterior distribution where the likelihood and prior are conjugate.
+
+    Currently supported conjugate pairs are:
+    - (Gaussian, Gamma)
+    - (GMRF, Gamma)
+    - (RegularizedGaussian, Gamma) with nonnegativity constraints only
+
+    For more information on conjugate pairs, see https://en.wikipedia.org/wiki/Conjugate_prior.
+
+    For implicit regularized Gaussians see:
+    
+    [1] Everink, Jasper M., Yiqiu Dong, and Martin S. Andersen. "Bayesian inference with projected densities." SIAM/ASA Journal on Uncertainty Quantification 11.3 (2023): 1025-1043.
+
+    """
+
+    def __init__(self, target: Posterior):
+        if not isinstance(target.likelihood.distribution, (Gaussian, GMRF, RegularizedGaussian, RegularizedGMRF)):
+            raise ValueError("Conjugate sampler only works with a Gaussian-type likelihood function")
+        if not isinstance(target.prior, Gamma):
+            raise ValueError("Conjugate sampler only works with Gamma prior")
+        if not target.prior.dim == 1:
+            raise ValueError("Conjugate sampler only works with univariate Gamma prior")
+            
+        if isinstance(target.likelihood.distribution, (RegularizedGaussian, RegularizedGMRF)) and (target.likelihood.distribution.preset["constraint"] not in ["nonnegativity"] or target.likelihood.distribution.preset["regularization"] is not None) :
+               raise ValueError("Conjugate sampler only works implicit regularized Gaussian likelihood with nonnegativity constraints")
+        
+        self.target = target
+
+    def step(self, x=None):
+        # Extract variables
+        b = self.target.likelihood.data                                 #mu
+        m = self._calc_m_for_Gaussians(b)                               #n
+        Ax = self.target.likelihood.distribution.mean                   #x_i
+        L = self.target.likelihood.distribution(np.array([1])).sqrtprec #L
+        alpha = self.target.prior.shape                                 #alpha
+        beta = self.target.prior.rate                                   #beta
+
+        # Create Gamma distribution and sample
+        dist = Gamma(shape=m/2+alpha,rate=.5*np.linalg.norm(L@(Ax-b))**2+beta)
+
+        return dist.sample()
+
+    def _calc_m_for_Gaussians(self, b):
+        """ Helper method to calculate m parameter for Gaussian-Gamma conjugate pair. """
+        if isinstance(self.target.likelihood.distribution, (Gaussian, GMRF)):
+            return len(b)
+        elif isinstance(self.target.likelihood.distribution, (RegularizedGaussian, RegularizedGMRF)):
+            return np.count_nonzero(b) # See 

cuqi/legacy/sampler/_conjugate_approx.py

@@ -0,0 +1,52 @@
+from cuqi.distribution import Posterior, LMRF, Gamma
+import numpy as np
+import scipy as sp
+
+class ConjugateApprox: # TODO: Subclass from Sampler once updated
+    """ Approximate Conjugate sampler
+
+    Sampler for sampling a posterior distribution where the likelihood and prior can be approximated
+    by a conjugate pair.
+
+    Currently supported pairs are:
+    - (LMRF, Gamma): Approximated by (Gaussian, Gamma)
+
+    For more information on conjugate pairs, see https://en.wikipedia.org/wiki/Conjugate_prior.
+
+    """
+
+    
+    def __init__(self, target: Posterior):
+        if not isinstance(target.likelihood.distribution, LMRF):
+            raise ValueError("Conjugate sampler only works with Laplace diff likelihood function")
+        if not isinstance(target.prior, Gamma):
+            raise ValueError("Conjugate sampler only works with Gamma prior")
+        self.target = target
+
+    def step(self, x=None):
+        # Extract variables
+        # Here we approximate the Laplace diff with a Gaussian
+
+        # Extract diff_op from target likelihood
+        D = self.target.likelihood.distribution._diff_op
+        n = D.shape[0]
+
+        # Gaussian approximation of LMRF prior as function of x_k
+        # See Uribe et al. (2022) for details
+        # Current has a zero mean assumption on likelihood! TODO
+        beta=1e-5
+        def Lk_fun(x_k):
+            dd =  1/np.sqrt((D @ x_k)**2 + beta*np.ones(n))
+            W = sp.sparse.diags(dd)
+            return W.sqrt() @ D
+
+        x = self.target.likelihood.data                 #x
+        d = len(x)                                      #d
+        Lx = Lk_fun(x)@x                                #Lx
+        alpha = self.target.prior.shape                 #alpha
+        beta = self.target.prior.rate                   #beta
+
+        # Create Gamma distribution and sample
+        dist = Gamma(shape=d+alpha, rate=np.linalg.norm(Lx)**2+beta)
+
+        return dist.sample()

cuqi/legacy/sampler/_cwmh.py

@@ -0,0 +1,196 @@
+import numpy as np
+import cuqi
+from cuqi.legacy.sampler import ProposalBasedSampler
+
+
+class CWMH(ProposalBasedSampler):
+    """Component-wise Metropolis Hastings sampler.
+
+    Allows sampling of a target distribution by a component-wise random-walk sampling of a proposal distribution along with an accept/reject step.
+
+    Parameters
+    ----------
+
+    target : `cuqi.distribution.Distribution` or lambda function
+        The target distribution to sample. Custom logpdfs are supported by using a :class:`cuqi.distribution.UserDefinedDistribution`.
+    
+    proposal : `cuqi.distribution.Distribution` or callable method
+        The proposal to sample from. If a callable method it should provide a single independent sample from proposal distribution. Defaults to a Gaussian proposal.  *Optional*.
+
+    scale : float
+        Scale parameter used to define correlation between previous and proposed sample in random-walk.  *Optional*.
+
+    x0 : ndarray
+        Initial parameters. *Optional*
+
+    dim : int
+        Dimension of parameter space. Required if target and proposal are callable functions. *Optional*.
+
+    callback : callable, *Optional*
+        If set this function will be called after every sample.
+        The signature of the callback function is `callback(sample, sample_index)`,
+        where `sample` is the current sample and `sample_index` is the index of the sample.
+        An example is shown in demos/demo31_callback.py.
+
+    Example
+    -------
+    .. code-block:: python
+
+        # Parameters
+        dim = 5 # Dimension of distribution
+        mu = np.arange(dim) # Mean of Gaussian
+        std = 1 # standard deviation of Gaussian
+
+        # Logpdf function
+        logpdf_func = lambda x: -1/(std**2)*np.sum((x-mu)**2)
+
+        # Define distribution from logpdf as UserDefinedDistribution (sample and gradients also supported as inputs to UserDefinedDistribution)
+        target = cuqi.distribution.UserDefinedDistribution(dim=dim, logpdf_func=logpdf_func)
+
+        # Set up sampler
+        sampler = cuqi.legacy.sampler.CWMH(target, scale=1)
+
+        # Sample
+        samples = sampler.sample(2000)
+
+    """
+    def __init__(self, target,  proposal=None, scale=1, x0=None, dim = None, **kwargs):
+        super().__init__(target, proposal=proposal, scale=scale,  x0=x0, dim=dim, **kwargs)
+        
+    @ProposalBasedSampler.proposal.setter 
+    def proposal(self, value):
+        fail_msg = "Proposal should be either None, cuqi.distribution.Distribution conditioned only on 'location' and 'scale', lambda function, or cuqi.distribution.Normal conditioned only on 'mean' and 'std'"
+
+        if value is None:
+            self._proposal = cuqi.distribution.Normal(mean = lambda location:location,std = lambda scale:scale, geometry=self.dim)
+
+        elif isinstance(value, cuqi.distribution.Distribution) and sorted(value.get_conditioning_variables())==['location','scale']:
+            self._proposal = value
+
+        elif isinstance(value, cuqi.distribution.Normal) and sorted(value.get_conditioning_variables())==['mean','std']:
+            self._proposal = value(mean = lambda location:location, std = lambda scale:scale)
+
+        elif not isinstance(value, cuqi.distribution.Distribution) and callable(value):
+            self._proposal = value
+
+        else:
+            raise ValueError(fail_msg)
+
+
+    def _sample(self, N, Nb):
+        Ns = N+Nb   # number of simulations
+
+        # allocation
+        samples = np.empty((self.dim, Ns))
+        target_eval = np.empty(Ns)
+        acc = np.zeros((self.dim, Ns), dtype=int)
+
+        # initial state    
+        samples[:, 0] = self.x0
+        target_eval[0] = self.target.logd(self.x0)
+        acc[:, 0] = np.ones(self.dim)
+
+        # run MCMC
+        for s in range(Ns-1):
+            # run component by component
+            samples[:, s+1], target_eval[s+1], acc[:, s+1] = self.single_update(samples[:, s], target_eval[s])
+
+            self._print_progress(s+2,Ns) #s+2 is the sample number, s+1 is index assuming x0 is the first sample
+            self._call_callback(samples[:, s+1], s+1)
+
+        # remove burn-in
+        samples = samples[:, Nb:]
+        target_eval = target_eval[Nb:]
+        acccomp = acc[:, Nb:].mean(axis=1)   
+        print('\nAverage acceptance rate all components:', acccomp.mean(), '\n')
+        
+        return samples, target_eval, acccomp
+
+    def _sample_adapt(self, N, Nb):
+        # this follows the vanishing adaptation Algorithm 4 in:
+        # Andrieu and Thoms (2008) - A tutorial on adaptive MCMC
+        Ns = N+Nb   # number of simulations
+
+        # allocation
+        samples = np.empty((self.dim, Ns))
+        target_eval = np.empty(Ns)
+        acc = np.zeros((self.dim, Ns), dtype=int)
+
+        # initial state
+        samples[:, 0] = self.x0
+        target_eval[0] = self.target.logd(self.x0)
+        acc[:, 0] = np.ones(self.dim)
+
+        # initial adaptation params 
+        Na = int(0.1*N)                                        # iterations to adapt
+        hat_acc = np.empty((self.dim, int(np.floor(Ns/Na))))     # average acceptance rate of the chains
+        lambd = np.empty((self.dim, int(np.floor(Ns/Na)+1)))     # scaling parameter \in (0,1)
+        lambd[:, 0] = self.scale
+        star_acc = 0.21/self.dim + 0.23    # target acceptance rate RW
+        i, idx = 0, 0
+
+        # run MCMC
+        for s in range(Ns-1):
+            # run component by component
+            samples[:, s+1], target_eval[s+1], acc[:, s+1] = self.single_update(samples[:, s], target_eval[s])
+            
+            # adapt prop spread of each component using acc of past samples
+            if ((s+1) % Na == 0):
+                # evaluate average acceptance rate
+                hat_acc[:, i] = np.mean(acc[:, idx:idx+Na], axis=1)
+
+                # compute new scaling parameter
+                zeta = 1/np.sqrt(i+1)   # ensures that the variation of lambda(i) vanishes
+                lambd[:, i+1] = np.exp(np.log(lambd[:, i]) + zeta*(hat_acc[:, i]-star_acc))  
+
+                # update parameters
+                self.scale = np.minimum(lambd[:, i+1], np.ones(self.dim))
+
+                # update counters
+                i += 1
+                idx += Na
+
+            # display iterations 
+            self._print_progress(s+2,Ns) #s+2 is the sample number, s+1 is index assuming x0 is the first sample
+            self._call_callback(samples[:, s+1], s+1)
+            
+        # remove burn-in
+        samples = samples[:, Nb:]
+        target_eval = target_eval[Nb:]
+        acccomp = acc[:, Nb:].mean(axis=1)
+        print('\nAverage acceptance rate all components:', acccomp.mean(), '\n')
+        
+        return samples, target_eval, acccomp
+
+    def single_update(self, x_t, target_eval_t):
+        if isinstance(self.proposal,cuqi.distribution.Distribution):
+            x_i_star = self.proposal(location= x_t, scale = self.scale).sample()
+        else:
+            x_i_star = self.proposal(x_t, self.scale) 
+        x_star = x_t.copy()
+        acc = np.zeros(self.dim)
+
+        for j in range(self.dim):
+            # propose state
+            x_star[j] = x_i_star[j]
+
+            # evaluate target
+            target_eval_star = self.target.logd(x_star)
+
+            # ratio and acceptance probability
+            ratio = target_eval_star - target_eval_t  # proposal is symmetric
+            alpha = min(0, ratio)
+
+            # accept/reject
+            u_theta = np.log(np.random.rand())
+            if (u_theta <= alpha):
+                x_t[j] = x_i_star[j]
+                target_eval_t = target_eval_star
+                acc[j] = 1
+            else:
+                pass
+                # x_t[j]       = x_t[j]
+                # target_eval_t = target_eval_t
+            x_star = x_t.copy()
+        #
+        return x_t, target_eval_t, acc