qpytorch 0.1__py3-none-any.whl

qpytorch/mlls/inducing_point_kernel_added_loss_term.py

@@ -0,0 +1,69 @@
+#!/usr/bin/env python3
+
+from typing import Union
+import torch
+
+from ..distributions import MultivariateNormal, MultivariateQExponential
+from ..likelihoods import GaussianLikelihood, MultitaskGaussianLikelihood, QExponentialLikelihood, MultitaskQExponentialLikelihood
+from gpytorch.mlls.added_loss_term import AddedLossTerm
+
+
+class InducingPointKernelAddedLossTerm(AddedLossTerm):
+    r"""
+    An added loss term that computes the additional "regularization trace term" of the SGPR (SQEPR) objective function.
+
+    .. math::
+        Gaussian: -\frac{1}{2 \sigma^2} \text{Tr} \left( \mathbf K_{\mathbf X \mathbf X} - \mathbf Q \right)
+    .. math::
+        Q-Exponential: \frac{d}{2}\left(-\log\sigma^2 +\left(\frac{q}{2}-1\right)\log r\right) -\frac{1}{2}r^{\frac{q}{2}}, 
+        r = \frac{1}{\sigma^2}\text{Tr} \left( \mathbf K_{\mathbf X \mathbf X} - \mathbf Q \right)
+
+
+    where :math:`\mathbf Q = \mathbf K_{\mathbf X \mathbf Z} \mathbf K_{\mathbf Z \mathbf Z}^{-1}
+    \mathbf K_{\mathbf Z \mathbf X}` is the Nystrom approximation of :math:`\mathbf K_{\mathbf X \mathbf X}`
+    given by inducing points :math:`\mathbf Z`, :math:`\sigma^2` is the observational noise
+    of the Gaussian (Q-Exponential) likelihood, and :math:`d` is the dimensions being summed over, 
+    i.e. :math:`N` for likelihood or :math:`ND` for multi-task likelihood.
+
+    See `Titsias, 2009`_, Eq. 9 for more more information.
+
+    :param prior_dist: A multivariate normal :math:`\mathcal N ( \mathbf 0, \mathbf K_{\mathbf X \mathbf X} )`
+        or q-exponential :math:`\mathcal Q ( \mathbf 0, \mathbf K_{\mathbf X \mathbf X} )`
+        with covariance matrix :math:`\mathbf K_{\mathbf X \mathbf X}`.
+    :param variational_dist: A multivariate normal :math:`\mathcal N ( \mathbf 0, \mathbf Q)`
+        or or q-exponential :math:`\mathcal Q ( \mathbf 0, \mathbf Q)`
+        with covariance matrix :math:`\mathbf Q = \mathbf K_{\mathbf X \mathbf Z}
+        \mathbf K_{\mathbf Z \mathbf Z}^{-1} \mathbf K_{\mathbf Z \mathbf X}`.
+    :param likelihood: The Gaussian (QExponential) likelihood with observational noise :math:`\sigma^2`.
+
+    .. _Titsias, 2009:
+        https://proceedings.mlr.press/v9/titsias10a/titsias10a.pdf
+    """
+
+    def __init__(
+        self, prior_dist: Union[MultivariateNormal, MultivariateQExponential], 
+        variational_dist: Union[MultivariateNormal, MultivariateQExponential], 
+        likelihood: Union[GaussianLikelihood, QExponentialLikelihood],
+    ):
+        self.prior_dist = prior_dist
+        self.variational_dist = variational_dist
+        self.likelihood = likelihood
+
+    def loss(self, *params) -> torch.Tensor:
+        prior_covar = self.prior_dist.lazy_covariance_matrix
+        variational_covar = self.variational_dist.lazy_covariance_matrix
+        diag = prior_covar.diagonal(dim1=-1, dim2=-2) - variational_covar.diagonal(dim1=-1, dim2=-2)
+        shape = prior_covar.shape[:-1]
+        if isinstance(self.likelihood, (MultitaskGaussianLikelihood, MultitaskQExponentialLikelihood)):
+            shape = torch.Size([*shape, 1])
+            diag = diag.unsqueeze(-1)
+        noise_diag = self.likelihood._shaped_noise_covar(shape, *params).diagonal(dim1=-1, dim2=-2)
+        if isinstance(self.likelihood, (MultitaskGaussianLikelihood, MultitaskQExponentialLikelihood)):
+            noise_diag = noise_diag.reshape(*shape[:-1], -1)
+            r = (diag / noise_diag).sum(dim=[-1, -2])
+        else:
+            r = (diag / noise_diag).sum(dim=-1)
+        res = -0.5 * r**(self.likelihood.power/2. if hasattr(self.likelihood,'power') else 1)
+        if 'QExponential' in self.likelihood.__class__.__name__:
+            if self.likelihood.power!=2: res += -0.5 * noise_diag.log().sum() + torch.tensor(noise_diag.shape[-2:]).prod()/2. * (self.likelihood.power/2.-1) * r.log()
+        return res

qpytorch/mlls/kl_qexponential_added_loss_term.py

@@ -0,0 +1,41 @@
+#!/usr/bin/env python3
+
+from torch.distributions import kl_divergence
+
+from ..distributions import MultivariateQExponential
+from gpytorch.mlls.added_loss_term import AddedLossTerm
+
+
+class KLQExponentialAddedLossTerm(AddedLossTerm):
+    r"""
+    This class is used by variational QEPLVM models.
+    It adds the KL divergence between two multivariate Q-Exponential distributions:
+    scaled by the size of the data and the number of output dimensions.
+
+    .. math::
+
+        D_\text{KL} \left( q(\mathbf x) \Vert p(\mathbf x) \right)
+
+
+    :param q_x: The QEP distribution :math:`q(\mathbf x)`.
+    :param p_x: The QEP distribution :math:`p(\mathbf x)`.
+    :param n: Size of the latent space.
+    :param data_dim: Dimensionality of the :math:`\mathbf Y` values.
+    """
+
+    def __init__(self, q_x: MultivariateQExponential, p_x: MultivariateQExponential, n: int, data_dim: int):
+        super().__init__()
+        self.q_x = q_x
+        self.p_x = p_x
+        self.n = n
+        self.data_dim = data_dim
+
+    def loss(self):
+        kl_per_latent_dim = kl_divergence(self.q_x, self.p_x).sum(axis=0)  # vector of size latent_dim
+        kl_per_point = kl_per_latent_dim.sum() / self.n  # scalar
+        # inside the forward method of variational ELBO,
+        # the added loss terms are expanded (using add_) to take the same
+        # shape as the log_lik term (has shape data_dim)
+        # so they can be added together. Hence, we divide by data_dim to avoid
+        # overcounting the kl term
+        return kl_per_point / self.data_dim

qpytorch/mlls/leave_one_out_pseudo_likelihood.py

@@ -0,0 +1,73 @@
+#!/usr/bin/env python3
+import math
+from typing import Union
+import torch
+from torch import Tensor
+
+from ..distributions import MultivariateNormal, MultivariateQExponential
+from .exact_marginal_log_likelihood import ExactMarginalLogLikelihood
+
+
+class LeaveOneOutPseudoLikelihood(ExactMarginalLogLikelihood):
+    r"""
+    The leave one out cross-validation (LOO-CV) likelihood from RW 5.4.2 for an exact Gaussian (Q-Exponential) process with a
+    Gaussian (Q-Exponential) likelihood. This offers an alternative to the exact marginal log likelihood where we
+    instead maximize the sum of the leave one out log probabilities :math:`\log p(y_i | X, y_{-i}, \theta)`.
+
+    Naively, this will be O(n^4) with Cholesky as we need to compute `n` Cholesky factorizations. Fortunately,
+    given the Cholesky factorization of the full kernel matrix (without any points removed), we can compute
+    both the mean and variance of each removed point via a bordered system formulation making the total
+    complexity O(n^3).
+
+    The LOO-CV approach can be more robust against model mis-specification as it gives an estimate for the
+    (log) predictive probability, whether or not the assumptions of the model is fulfilled.
+
+    .. note::
+        This module will not work with anything other than a :obj:`~qpytorch.likelihoods.GaussianLikelihood`
+        (:obj:`~qpytorch.likelihoods.QExponentialLikelihood`) and a :obj:`~gpytorch.models.ExactGP` (:obj:`~qpytorch.models.ExactQEP`). 
+        It also cannot be used in conjunction with stochastic optimization.
+
+    :param ~qpytorch.likelihoods.GaussianLikelihood (~qpytorch.likelihoods.QExponentialLikelihood) likelihood: The Gaussian (Q-Exponential) likelihood for the model
+    :param ~gpytorch.models.ExactGP (~qpytorch.models.ExactQEP) model: The exact GP (QEP) model
+
+    Example:
+        >>> # model is a qpytorch.models.ExactGP or qpytorch.models.ExactQEP
+        >>> # likelihood is a qpytorch.likelihoods.Likelihood
+        >>> loocv = qpytorch.mlls.LeaveOneOutPseudoLikelihood(likelihood, model)
+        >>>
+        >>> output = model(train_x)
+        >>> loss = -loocv(output, train_y)
+        >>> loss.backward()
+    """
+
+    def __init__(self, likelihood, model):
+        super().__init__(likelihood=likelihood, model=model)
+        self.likelihood = likelihood
+        self.model = model
+
+    def forward(self, function_dist: Union[MultivariateNormal, MultivariateQExponential], target: Tensor, *params) -> Tensor:
+        r"""
+        Computes the leave one out likelihood given :math:`p(\mathbf f)` and :math:`\mathbf y`
+
+        :param ~gpytorch.distributions.MultivariateNormal (~qpytorch.distributions.MultivariateQExponential) 
+            output: the outputs of the latent function (the :obj:`~gpytorch.models.GP` or :obj:`~qpytorch.models.QEP`)
+        :param torch.Tensor target: :math:`\mathbf y` The target values
+        :param dict kwargs: Additional arguments to pass to the likelihood's forward function.
+        """
+        output = self.likelihood(function_dist, *params)
+        m, L = output.mean, output.lazy_covariance_matrix.cholesky(upper=False)
+        m = m.reshape(*target.shape)
+        identity = torch.eye(*L.shape[-2:], dtype=m.dtype, device=m.device)
+        sigma2 = 1.0 / L._cholesky_solve(identity, upper=False).diagonal(dim1=-1, dim2=-2)  # 1 / diag(inv(K))
+        mu = target - L._cholesky_solve((target - m).unsqueeze(-1), upper=False).squeeze(-1) * sigma2
+        term1 = -0.5 * sigma2.log()
+        power = getattr(self.likelihood, 'power', torch.tensor(2.0))
+        term2 = -0.5 * (target - mu).abs()**power / sigma2**(power/2.)
+        res = (term1 + term2).sum(dim=-1)
+        if power!=2: res += (power/2.-1) * ((target - mu).abs().log() + term1).sum(dim=-1)
+
+        res = self._add_other_terms(res, params)
+
+        # Scale by the amount of data we have and then add on the scaled constant
+        num_data = target.size(-1)
+        return res.div_(num_data) - 0.5 * math.log(2 * math.pi)

qpytorch/mlls/marginal_log_likelihood.py

@@ -0,0 +1,48 @@
+#!/usr/bin/env python3
+
+from ..models import GP, QEP
+from ..module import Module
+
+
+class MarginalLogLikelihood(Module):
+    r"""
+    These are modules to compute (or approximate/bound) the marginal log likelihood
+    (MLL) of the GP (QEP) model when applied to data.  I.e., given a GP :math:`f \sim
+    \mathcal{GP}(\mu, K)` or QEP :math:`f \sim \mathcal{QEP}(\mu, K)`, and 
+    data :math:`\mathbf X, \mathbf y`, these modules compute/approximate
+
+    .. math::
+
+       \begin{equation*}
+          \mathcal{L} = p_f(\mathbf y \! \mid \! \mathbf X)
+          = \int p \left( \mathbf y \! \mid \! f(\mathbf X) \right) \: p(f(\mathbf X) \! \mid \! \mathbf X) \: d f
+       \end{equation*}
+
+    This is computed exactly when the GP (QEP) inference is computed exactly (e.g. regression w/ a Gaussian (Q-Exponential) likelihood).
+    It is approximated/bounded for GP (QEP) models that use approximate inference.
+
+    These models are typically used as the "loss" functions for GP (QEP) models (though note that the output of
+    these functions must be negated for optimization).
+    """
+
+    def __init__(self, likelihood, model):
+        super(MarginalLogLikelihood, self).__init__()
+        if not isinstance(model, (GP, QEP)):
+            raise RuntimeError(
+                "All MarginalLogLikelihood objects must be given a GP (QEP) object as a model. If you are "
+                "using a more complicated model involving a GP (QEP), pass the underlying GP (QEP) object as the "
+                "model, not a full PyTorch module."
+            )
+        self.likelihood = likelihood
+        self.model = model
+
+    def forward(self, output, target, **kwargs):
+        r"""
+        Computes the MLL given :math:`p(\mathbf f)` and `\mathbf y`
+
+        :param ~gpytorch.distributions.MultivariateNormal or ~qpytorch.distributions.MultivariateQExponential 
+            output: the outputs of the latent function (the :obj:`~gpytorch.models.GP` or :obj:`~qpytorch.models.QEP`)
+        :param torch.Tensor target: :math:`\mathbf y` The target values
+        :param dict kwargs: Additional arguments to pass to the likelihood's forward function.
+        """
+        raise NotImplementedError

qpytorch/mlls/predictive_log_likelihood.py

@@ -0,0 +1,76 @@
+#!/usr/bin/env python3
+
+from ._approximate_mll import _ApproximateMarginalLogLikelihood
+
+
+class PredictiveLogLikelihood(_ApproximateMarginalLogLikelihood):
+    r"""
+    An alternative objective function for approximate GPs (QEPs), proposed in `Jankowiak et al., 2020`_.
+    It typically produces better predictive variances than the :obj:`qpytorch.mlls.VariationalELBO` objective.
+
+    .. math::
+
+       \begin{align*}
+          \mathcal{L}_\text{ELBO} &=
+          \mathbb{E}_{p_\text{data}( y, \mathbf x )} \left[
+            \log p( y \! \mid \! \mathbf x)
+          \right] - \beta \: \text{KL} \left[ q( \mathbf u) \Vert p( \mathbf u) \right]
+          \\
+          &\approx \sum_{i=1}^N \log \mathbb{E}_{q(\mathbf u)} \left[
+            \int p( y_i \! \mid \! f_i) p(f_i \! \mid \! \mathbf u, \mathbf x_i) \: d f_i
+          \right] - \beta \: \text{KL} \left[ q( \mathbf u) \Vert p( \mathbf u) \right]
+       \end{align*}
+
+    where :math:`N` is the total number of datapoints, :math:`q(\mathbf u)` is the variational distribution for
+    the inducing function values, and :math:`p(\mathbf u)` is the prior distribution for the inducing function
+    values.
+
+    :math:`\beta` is a scaling constant that reduces the regularization effect of the KL
+    divergence. Setting :math:`\beta=1` (default) results in an objective that can be motivated by a connection
+    to Stochastic Expectation Propagation (see `Jankowiak et al., 2020`_ for details).
+
+    .. note::
+        This objective is very similar to the variational ELBO.
+        The only difference is that the :math:`log` occurs *outside* the expectation :math:`\mathbb{E}_{q(\mathbf u)}`.
+        This difference results in very different predictive performance (see `Jankowiak et al., 2020`_).
+
+    :param ~qpytorch.likelihoods.Likelihood likelihood: The likelihood for the model
+    :param ~gpytorch.models.ApproximateGP (~qpytorch.models.ApproximateQEP) model: The approximate GP (QEP) model
+    :param int num_data: The total number of training data points (necessary for SGD)
+    :param float beta: (optional, default=1.) A multiplicative factor for the KL divergence term.
+        Setting it to anything less than 1 reduces the regularization effect of the model
+        (similarly to what was proposed in `the beta-VAE paper`_).
+    :param bool combine_terms: (default=True): Whether or not to sum the
+        expected NLL with the KL terms (default True)
+
+    Example:
+        >>> # model is a qpytorch.models.ApproximateGP or qpytorch.models.ApproximateQEP 
+        >>> # likelihood is a qpytorch.likelihoods.Likelihood
+        >>> mll = qpytorch.mlls.PredictiveLogLikelihood(likelihood, model, num_data=100, beta=0.5)
+        >>>
+        >>> output = model(train_x)
+        >>> loss = -mll(output, train_y)
+        >>> loss.backward()
+
+    .. _Jankowiak et al., 2020:
+        https://arxiv.org/abs/1910.07123
+    """
+
+    def _log_likelihood_term(self, approximate_dist_f, target, **kwargs):
+        return self.likelihood.log_marginal(target, approximate_dist_f, **kwargs).sum(-1)
+
+    def forward(self, approximate_dist_f, target, **kwargs):
+        r"""
+        Computes the predictive cross entropy given :math:`q(\mathbf f)` and :math:`\mathbf y`.
+        Calling this function will call the likelihood's
+        :meth:`~qpytorch.likelihoods.Likelihood.forward` function.
+
+        :param ~gpytorch.distributions.MultivariateNormal variational_dist_f: :math:`q(\mathbf f)`
+            the outputs of the latent function (the :obj:`gpytorch.models.ApproximateGP` or :obj:`qpytorch.models.ApproximateQEP`)
+        :param torch.Tensor target: :math:`\mathbf y` The target values
+        :param kwargs: Additional arguments passed to the
+            likelihood's :meth:`~qpytorch.likelihoods.Likelihood.forward` function.
+        :rtype: torch.Tensor
+        :return: Predictive log likelihood. Output shape corresponds to batch shape of the model/input data.
+        """
+        return super().forward(approximate_dist_f, target, **kwargs)

qpytorch/mlls/sum_marginal_log_likelihood.py

@@ -0,0 +1,40 @@
+#! /usr/bin/env python3
+
+from torch.nn import ModuleList
+
+from . import ExactMarginalLogLikelihood, MarginalLogLikelihood
+from gpytorch.utils.generic import length_safe_zip
+
+
+class SumMarginalLogLikelihood(MarginalLogLikelihood):
+    """Sum of marginal log likelihoods, to be used with Multi-Output models.
+
+    Args:
+        likelihood: A MultiOutputLikelihood
+        model: A MultiOutputModel
+        mll_cls: The Marginal Log Likelihood class (default: ExactMarginalLogLikelihood)
+
+    In case the model outputs are independent/uncorrelated, this provides the MLL of the multi-output model.
+
+    """
+
+    def __init__(self, likelihood, model, mll_cls=ExactMarginalLogLikelihood):
+        super().__init__(model.likelihood, model)
+        self.mlls = ModuleList([mll_cls(mdl.likelihood, mdl) for mdl in model.models])
+
+    def forward(self, outputs, targets, *params):
+        """
+        Args:
+            outputs: (Iterable[MultivariateNormal/MultivariateQExponential]) - the outputs of the latent function
+            targets: (Iterable[Tensor]) - the target values
+            params: (Iterable[Iterable[Tensor]]) - the arguments to be passed through
+                (e.g. parameters in case of heteroskedastic likelihoods)
+        """
+        if len(params) == 0:
+            sum_mll = sum(mll(output, target) for mll, output, target in length_safe_zip(self.mlls, outputs, targets))
+        else:
+            sum_mll = sum(
+                mll(output, target, *iparams)
+                for mll, output, target, iparams in length_safe_zip(self.mlls, outputs, targets, params)
+            )
+        return sum_mll.div_(len(self.mlls))

qpytorch/mlls/variational_elbo.py

@@ -0,0 +1,77 @@
+#!/usr/bin/env python3
+
+from ._approximate_mll import _ApproximateMarginalLogLikelihood
+
+
+class VariationalELBO(_ApproximateMarginalLogLikelihood):
+    r"""
+    The variational evidence lower bound (ELBO). This is used to optimize
+    variational Gaussian (Q-Exponential) processes (with or without stochastic optimization).
+
+    .. math::
+
+       \begin{align*}
+          \mathcal{L}_\text{ELBO} &=
+          \mathbb{E}_{p_\text{data}( y, \mathbf x )} \left[
+            \mathbb{E}_{p(f \mid \mathbf u, \mathbf x) q(\mathbf u)} \left[  \log p( y \! \mid \! f) \right]
+          \right] - \beta \: \text{KL} \left[ q( \mathbf u) \Vert p( \mathbf u) \right]
+          \\
+          &\approx \sum_{i=1}^N \mathbb{E}_{q( f_i)} \left[
+            \log p( y_i \! \mid \! f_i) \right] - \beta \: \text{KL} \left[ q( \mathbf u) \Vert p( \mathbf u) \right]
+       \end{align*}
+
+    where :math:`N` is the number of datapoints, :math:`q(\mathbf u)` is the variational distribution for
+    the inducing function values, :math:`q(f_i)` is the marginal of
+    :math:`p(f_i \mid \mathbf u, \mathbf x_i) q(\mathbf u)`,
+    and :math:`p(\mathbf u)` is the prior distribution for the inducing function values.
+
+    :math:`\beta` is a scaling constant that reduces the regularization effect of the KL
+    divergence. Setting :math:`\beta=1` (default) results in the true variational ELBO.
+
+    For more information on this derivation, see `Scalable Variational Gaussian Process Classification`_
+    (Hensman et al., 2015).
+
+    :param ~qpytorch.likelihoods.Likelihood likelihood: The likelihood for the model
+    :param ~gpytorch.models.ApproximateGP (~qpytorch.models.ApproximateQEP) model: The approximate GP (QEP) model
+    :param int num_data: The total number of training data points (necessary for SGD)
+    :param float beta: (optional, default=1.) A multiplicative factor for the KL divergence term.
+        Setting it to 1 (default) recovers true variational inference
+        (as derived in `Scalable Variational Gaussian Process Classification`_).
+        Setting it to anything less than 1 reduces the regularization effect of the model
+        (similarly to what was proposed in `the beta-VAE paper`_).
+    :param bool combine_terms: (default=True): Whether or not to sum the
+        expected NLL with the KL terms (default True)
+
+    Example:
+        >>> # model is a qpytorch.models.ApproximateGP or qpytorch.models.ApproximateQEP
+        >>> # likelihood is a qpytorch.likelihoods.Likelihood
+        >>> mll = qpytorch.mlls.VariationalELBO(likelihood, model, num_data=100, beta=0.5)
+        >>>
+        >>> output = model(train_x)
+        >>> loss = -mll(output, train_y)
+        >>> loss.backward()
+
+    .. _Scalable Variational Gaussian Process Classification:
+        http://proceedings.mlr.press/v38/hensman15.pdf
+    .. _the beta-VAE paper:
+        https://openreview.net/pdf?id=Sy2fzU9gl
+    """
+
+    def _log_likelihood_term(self, variational_dist_f, target, **kwargs):
+        return self.likelihood.expected_log_prob(target, variational_dist_f, **kwargs).sum(-1)
+
+    def forward(self, variational_dist_f, target, **kwargs):
+        r"""
+        Computes the Variational ELBO given :math:`q(\mathbf f)` and :math:`\mathbf y`.
+        Calling this function will call the likelihood's :meth:`~qpytorch.likelihoods.Likelihood.expected_log_prob`
+        function.
+
+        :param ~gpytorch.distributions.MultivariateNormal (~qpytorch.distributions.MultivariateQExponential) variational_dist_f: :math:`q(\mathbf f)`
+            the outputs of the latent function (the :obj:`gpytorch.models.ApproximateGP` or :obj:`qpytorch.models.ApproximateQEP`)
+        :param torch.Tensor target: :math:`\mathbf y` The target values
+        :param kwargs: Additional arguments passed to the
+            likelihood's :meth:`~qpytorch.likelihoods.Likelihood.expected_log_prob` function.
+        :rtype: torch.Tensor
+        :return: Variational ELBO. Output shape corresponds to batch shape of the model/input data.
+        """
+        return super().forward(variational_dist_f, target, **kwargs)

qpytorch/models/__init__.py

@@ -0,0 +1,72 @@
+#!/usr/bin/env python3
+
+import warnings
+
+from gpytorch.models import deep_gps, gplvm
+from . import deep_qeps, exact_prediction_strategies, qeplvm, pyro
+from gpytorch.models.approximate_gp import ApproximateGP
+from gpytorch.models.exact_gp import ExactGP
+from gpytorch.models.gp import GP
+from .approximate_qep import ApproximateQEP
+from .exact_qep import ExactQEP
+from .qep import QEP
+from .model_list import AbstractModelList, IndependentModelList, UncorrelatedModelList
+from .pyro import PyroGP, PyroQEP
+
+# Alternative name for ApproximateGP, ApproximateQEP
+VariationalGP = ApproximateGP
+VariationalQEP = ApproximateQEP
+
+
+# Deprecated for 0.4 release
+class AbstractVariationalGP(ApproximateGP):
+    # Remove after 1.0
+    def __init__(self, *args, **kwargs):
+        warnings.warn("AbstractVariationalGP has been renamed to ApproximateGP.", DeprecationWarning)
+        super().__init__(*args, **kwargs)
+
+
+# Deprecated for 0.4 release
+class PyroVariationalGP(ApproximateGP):
+    # Remove after 1.0
+    def __init__(self, *args, **kwargs):
+        warnings.warn("PyroVariationalGP has been renamed to PyroGP.", DeprecationWarning)
+        super().__init__(*args, **kwargs)
+
+
+# Deprecated for 0.4 release
+class AbstractVariationalQEP(ApproximateQEP):
+    # Remove after 1.0
+    def __init__(self, *args, **kwargs):
+        warnings.warn("AbstractVariationalQEP has been renamed to ApproximateQEP.", DeprecationWarning)
+        super().__init__(*args, **kwargs)
+
+
+# Deprecated for 0.4 release
+class PyroVariationalQEP(ApproximateQEP):
+    # Remove after 1.0
+    def __init__(self, *args, **kwargs):
+        warnings.warn("PyroVariationalQEP has been renamed to PyroQEP.", DeprecationWarning)
+        super().__init__(*args, **kwargs)
+
+__all__ = [
+    "AbstractModelList",
+    "ApproximateGP",
+    "ApproximateQEP",
+    "ExactGP",
+    "ExactQEP",
+    "GP",
+    "QEP",
+    "IndependentModelList",
+    "PyroGP",
+    "PyroQEP",
+    "UncorrelatedModelList",
+    "VariationalGP",
+    "VariationalQEP",
+    "deep_gps",
+    "deep_qeps",
+    "gplvm",
+    "qeplvm",
+    "exact_prediction_strategies",
+    "pyro",
+]

qpytorch/models/approximate_qep.py

@@ -0,0 +1,115 @@
+#!/usr/bin/env python3
+
+from typing import Any, Optional
+
+from torch import Tensor
+
+from ..distributions import MultivariateQExponential
+from .exact_qep import ExactQEP
+
+from .qep import QEP
+from .pyro import _PyroMixin  # This will only contain functions if Pyro is installed
+
+
+class ApproximateQEP(QEP, _PyroMixin):
+    r"""
+    The base class for any Q-Exponential process latent function to be used in conjunction
+    with approximate inference (typically stochastic variational inference).
+    This base class can be used to implement most inducing point methods where the
+    variational parameters are learned directly.
+
+    :param ~qpytorch.variational._VariationalStrategy variational_strategy: The strategy that determines
+        how the model marginalizes over the variational distribution (over inducing points)
+        to produce the approximate posterior distribution (over data)
+
+    The :meth:`forward` function should describe how to compute the prior latent distribution
+    on a given input. Typically, this will involve a mean and kernel function.
+    The result must be a :obj:`~qpytorch.distributions.MultivariateQExponential`.
+
+    Example:
+        >>> class MyVariationalQEP(qpytorch.models.PyroQEP):
+        >>>     def __init__(self, power=torch.tensor(1.0), variational_strategy):
+        >>>         super().__init__(variational_strategy)
+        >>>         self.mean_module = qpytorch.means.ZeroMean()
+        >>>         self.covar_module = qpytorch.kernels.ScaleKernel(qpytorch.kernels.RBFKernel())
+        >>>         self.power = power
+        >>>
+        >>>     def forward(self, x):
+        >>>         mean = self.mean_module(x)
+        >>>         covar = self.covar_module(x)
+        >>>         return qpytorch.distributions.MultivariateQExponential(mean, covar, self.power)
+        >>>
+        >>> # variational_strategy = ...
+        >>> model = MyVariationalQEP(variational_strategy)
+        >>> likelihood = qpytorch.likelihoods.QExponentialLikelihood()
+        >>>
+        >>> # optimization loop for variational parameters...
+        >>>
+        >>> # test_x = ...;
+        >>> model(test_x)  # Returns the approximate QEP latent function at test_x
+        >>> likelihood(model(test_x))  # Returns the (approximate) predictive posterior distribution at test_x
+    """
+
+    def __init__(self, variational_strategy):
+        super().__init__()
+
+        self.variational_strategy = variational_strategy
+
+    def forward(self, x: Tensor):
+        raise NotImplementedError
+
+    def pyro_guide(self, input: Tensor, beta: float = 1.0, name_prefix: str = ""):
+        r"""
+        (For Pyro integration only). The component of a `pyro.guide` that
+        corresponds to drawing samples from the latent QEP function.
+
+        :param input: The inputs :math:`\mathbf X`.
+        :param beta: (default=1.) How much to scale the :math:`\text{KL} [ q(\mathbf f) \Vert p(\mathbf f) ]`
+            term by.
+        :param name_prefix: (default="") A name prefix to prepend to pyro sample sites.
+        """
+        return super().pyro_guide(input, beta=beta, name_prefix=name_prefix)
+
+    def pyro_model(self, input: Tensor, beta: float = 1.0, name_prefix: str = "") -> Tensor:
+        r"""
+        (For Pyro integration only). The component of a `pyro.model` that
+        corresponds to drawing samples from the latent QEP function.
+
+        :param input: The inputs :math:`\mathbf X`.
+        :param beta: (default=1.) How much to scale the :math:`\text{KL} [ q(\mathbf f) \Vert p(\mathbf f) ]`
+            term by.
+        :param name_prefix: (default="") A name prefix to prepend to pyro sample sites.
+        :return: samples from :math:`q(\mathbf f)`
+        """
+        return super().pyro_model(input, beta=beta, name_prefix=name_prefix)
+
+    def get_fantasy_model(self, inputs: Tensor, targets: Tensor, **kwargs: Any) -> ExactQEP:
+        r"""
+        Returns a new QEP model that incorporates the specified inputs and targets as new training data using
+        online variational conditioning (OVC).
+
+        This function first casts the inducing points and variational parameters into pseudo-points before
+        returning an equivalent ExactQEP model with a specialized likelihood.
+
+        .. note::
+            If `targets` is a batch (e.g. `b x m`), then the QEP returned from this method will be a batch mode QEP.
+            If `inputs` is of the same (or lesser) dimension as `targets`, then it is assumed that the fantasy points
+            are the same for each target batch.
+
+        :param inputs: (`b1 x ... x bk x m x d` or `f x b1 x ... x bk x m x d`) Locations of fantasy
+            observations.
+        :param targets: (`b1 x ... x bk x m` or `f x b1 x ... x bk x m`) Labels of fantasy observations.
+        :return: An `ExactQEP` model with `n + m` training examples, where the `m` fantasy examples have been added
+            and all test-time caches have been updated.
+
+        Reference: "Conditioning Sparse Variational Gaussian Processes for Online Decision-Making,"
+            Maddox, Stanton, Wilson, NeurIPS, '21
+            https://papers.nips.cc/paper/2021/hash/325eaeac5bef34937cfdc1bd73034d17-Abstract.html
+
+        """
+        return self.variational_strategy.get_fantasy_model(inputs=inputs, targets=targets, **kwargs)
+
+    def __call__(self, inputs: Optional[Tensor], prior: bool = False, **kwargs) -> MultivariateQExponential:
+        if inputs is not None and inputs.dim() == 1:
+            inputs = inputs.unsqueeze(-1)
+        return self.variational_strategy(inputs, prior=prior, **kwargs)

qpytorch/models/deep_qeps/__init__.py

@@ -0,0 +1,22 @@
+#!/usr/bin/env python3
+
+import warnings
+
+from .deep_qep import DeepQEP, DeepQEPLayer, DeepLikelihood
+
+
+# Deprecated for 1.0 release
+class AbstractDeepQEP(DeepQEP):
+    def __init__(self, *args, **kwargs):
+        warnings.warn("AbstractDeepQEP has been renamed to DeepQEP.", DeprecationWarning)
+        super().__init__(*args, **kwargs)
+
+
+# Deprecated for 1.0 release
+class AbstractDeepQEPLayer(DeepQEPLayer):
+    def __init__(self, *args, **kwargs):
+        warnings.warn("AbstractDeepQEPLayer has been renamed to DeepQEPLayer.", DeprecationWarning)
+        super().__init__(*args, **kwargs)
+
+
+__all__ = ["DeepQEPLayer", "DeepQEP", "AbstractDeepQEPLayer", "AbstractDeepQEP", "DeepLikelihood"]