qpytorch 0.1__py3-none-any.whl

qpytorch/likelihoods/softmax_likelihood.py

@@ -0,0 +1,97 @@
+#!/usr/bin/env python3
+
+import warnings
+from typing import Any, Optional, Union
+
+import torch
+from torch import Tensor
+from torch.distributions import Categorical, Distribution
+
+from ..distributions import base_distributions, MultitaskMultivariateNormal, MultitaskMultivariateQExponential
+from ..priors import Prior
+from .likelihood import Likelihood
+
+
+class SoftmaxLikelihood(Likelihood):
+    r"""
+    Implements the Softmax (multiclass) likelihood used for GP (QEP) classification.
+
+    .. math::
+        p(\mathbf y \mid \mathbf f) = \text{Softmax} \left( \mathbf W \mathbf f \right)
+
+    :math:`\mathbf W` is a set of linear mixing weights applied to the latent functions :math:`\mathbf f`.
+
+    :param num_features: Dimensionality of latent function :math:`\mathbf f`.
+    :param num_classes: Number of classes.
+    :param mixing_weights: (Default: `True`) Whether to learn a linear mixing weight :math:`\mathbf W` applied to
+        the latent function :math:`\mathbf f`. If `False`, then :math:`\mathbf W = \mathbf I`.
+    :param mixing_weights_prior: Prior to use over the mixing weights :math:`\mathbf W`.
+
+    :ivar torch.Tensor mixing_weights: (Optional) mixing weights.
+    """
+
+    def __init__(
+        self,
+        num_features: Optional[int] = None,
+        num_classes: int = None,  # pyre-fixme[9]
+        mixing_weights: bool = True,
+        mixing_weights_prior: Optional[Prior] = None,
+    ) -> None:
+        super().__init__()
+        if num_classes is None:
+            raise ValueError("num_classes is required")
+        self.num_classes = num_classes
+        if mixing_weights:
+            if num_features is None:
+                raise ValueError("num_features is required with mixing weights")
+            self.num_features: int = num_features
+            self.register_parameter(
+                name="mixing_weights",
+                parameter=torch.nn.Parameter(torch.randn(num_classes, num_features).div_(num_features)),
+            )
+            if mixing_weights_prior is not None:
+                self.register_prior("mixing_weights_prior", mixing_weights_prior, "mixing_weights")
+        else:
+            self.num_features = num_classes
+            self.mixing_weights: Optional[torch.nn.Parameter] = None
+
+    def forward(self, function_samples: Tensor, *params: Any, **kwargs: Any) -> Categorical:
+        num_data, num_features = function_samples.shape[-2:]
+
+        # Catch legacy mode
+        if num_data == self.num_features:
+            warnings.warn(
+                "The input to SoftmaxLikelihood should be a MultitaskMultivariateNormal or MultitaskMultivariateQExponential (num_data x num_tasks). "
+                "Batch MultivariateNormal inputs (num_tasks x num_data) will be deprectated.",
+                DeprecationWarning,
+            )
+            function_samples = function_samples.transpose(-1, -2)
+            num_data, num_features = function_samples.shape[-2:]
+
+        if num_features != self.num_features:
+            raise RuntimeError("There should be %d features" % self.num_features)
+
+        if self.mixing_weights is not None:
+            mixed_fs = function_samples @ self.mixing_weights.t()  # num_classes x num_data
+        else:
+            mixed_fs = function_samples
+        res = base_distributions.Categorical(logits=mixed_fs)
+        return res
+
+    def __call__(self, input: Union[Tensor, MultitaskMultivariateNormal, MultitaskMultivariateQExponential], *args: Any, **kwargs: Any) -> Distribution:
+        if isinstance(input, Distribution):
+            if not isinstance(input, MultitaskMultivariateNormal) and not hasattr(input, 'power'):
+                warnings.warn(
+                    "The input to SoftmaxLikelihood should be a MultitaskMultivariateNormal (num_data x num_tasks). "
+                    "Batch MultivariateNormal inputs (num_tasks x num_data) will be deprectated.",
+                    DeprecationWarning,
+                )
+                input = MultitaskMultivariateNormal.from_batch_mvn(input)
+            elif not isinstance(input, MultitaskMultivariateQExponential) and hasattr(input, 'power'):
+                warnings.warn(
+                    "The input to SoftmaxLikelihood should be a MultitaskMultivariateQExponential (num_data x num_tasks). "
+                    "Batch MultivariateQExponential inputs (num_tasks x num_data) will be deprectated.",
+                    DeprecationWarning,
+                )
+                input = MultitaskMultivariateQExponential.from_batch_qep(input)
+        return super().__call__(input, *args, **kwargs)

qpytorch/likelihoods/student_t_likelihood.py

@@ -0,0 +1,90 @@
+#!/usr/bin/env python3
+
+from typing import Any, Optional
+
+import torch
+from torch import Tensor
+from torch.distributions import StudentT
+
+from ..constraints import GreaterThan, Interval, Positive
+from ..distributions import base_distributions
+from ..priors import Prior
+from .likelihood import _OneDimensionalLikelihood
+
+
+class StudentTLikelihood(_OneDimensionalLikelihood):
+    r"""
+    A Student T likelihood/noise model for GP (QEP) regression.
+    It has two learnable parameters: :math:`\nu` - the degrees of freedom, and
+    :math:`\sigma^2` - the noise
+
+    :param batch_shape: The batch shape of the learned noise parameter (default: []).
+    :param noise_prior: Prior for noise parameter :math:`\sigma^2`.
+    :param noise_constraint: Constraint for noise parameter :math:`\sigma^2`.
+    :param deg_free_prior: Prior for deg_free parameter :math:`\nu`.
+    :param deg_free_constraint: Constraint for deg_free parameter :math:`\nu`.
+
+    :var torch.Tensor deg_free: :math:`\nu` parameter (degrees of freedom)
+    :var torch.Tensor noise: :math:`\sigma^2` parameter (noise)
+    """
+
+    def __init__(
+        self,
+        batch_shape: torch.Size = torch.Size([]),
+        deg_free_prior: Optional[Prior] = None,
+        deg_free_constraint: Optional[Interval] = None,
+        noise_prior: Optional[Prior] = None,
+        noise_constraint: Optional[Interval] = None,
+    ) -> None:
+        super().__init__()
+
+        if deg_free_constraint is None:
+            deg_free_constraint = GreaterThan(2)
+
+        if noise_constraint is None:
+            noise_constraint = Positive()
+
+        self.raw_deg_free = torch.nn.Parameter(torch.zeros(*batch_shape, 1))
+        self.raw_noise = torch.nn.Parameter(torch.zeros(*batch_shape, 1))
+
+        if noise_prior is not None:
+            self.register_prior("noise_prior", noise_prior, lambda m: m.noise, lambda m, v: m._set_noise(v))
+
+        self.register_constraint("raw_noise", noise_constraint)
+
+        if deg_free_prior is not None:
+            self.register_prior("deg_free_prior", deg_free_prior, lambda m: m.deg_free, lambda m, v: m._set_deg_free(v))
+
+        self.register_constraint("raw_deg_free", deg_free_constraint)
+
+        # Rough initialization
+        self.initialize(deg_free=7)
+
+    @property
+    def deg_free(self) -> Tensor:
+        return self.raw_deg_free_constraint.transform(self.raw_deg_free)
+
+    @deg_free.setter
+    def deg_free(self, value: Tensor) -> None:
+        self._set_deg_free(value)
+
+    def _set_deg_free(self, value: Tensor) -> None:
+        if not torch.is_tensor(value):
+            value = torch.as_tensor(value).to(self.raw_deg_free)
+        self.initialize(raw_deg_free=self.raw_deg_free_constraint.inverse_transform(value))
+
+    @property
+    def noise(self) -> Tensor:
+        return self.raw_noise_constraint.transform(self.raw_noise)
+
+    @noise.setter
+    def noise(self, value: Tensor) -> None:
+        self._set_noise(value)
+
+    def _set_noise(self, value: Tensor) -> None:
+        if not torch.is_tensor(value):
+            value = torch.as_tensor(value).to(self.raw_noise)
+        self.initialize(raw_noise=self.raw_noise_constraint.inverse_transform(value))
+
+    def forward(self, function_samples: Tensor, *args: Any, **kwargs: Any) -> StudentT:
+        return base_distributions.StudentT(df=self.deg_free, loc=function_samples, scale=self.noise.sqrt())

qpytorch/means/__init__.py

@@ -0,0 +1,23 @@
+#!/usr/bin/env python3
+
+from gpytorch.means.constant_mean import ConstantMean
+from gpytorch.means.constant_mean_grad import ConstantMeanGrad
+from gpytorch.means.constant_mean_gradgrad import ConstantMeanGradGrad
+from gpytorch.means.linear_mean import LinearMean
+from gpytorch.means.linear_mean_grad import LinearMeanGrad
+from gpytorch.means.linear_mean_gradgrad import LinearMeanGradGrad
+from gpytorch.means.mean import Mean
+from gpytorch.means.multitask_mean import MultitaskMean
+from gpytorch.means.zero_mean import ZeroMean
+
+__all__ = [
+    "Mean",
+    "ConstantMean",
+    "ConstantMeanGrad",
+    "ConstantMeanGradGrad",
+    "LinearMean",
+    "LinearMeanGrad",
+    "LinearMeanGradGrad",
+    "MultitaskMean",
+    "ZeroMean",
+]

qpytorch/metrics/__init__.py

@@ -0,0 +1,17 @@
+from gpytorch.metrics import (
+    mean_absolute_error,
+    mean_squared_error,
+    mean_standardized_log_loss,
+    negative_log_predictive_density,
+    quantile_coverage_error,
+    standardized_mean_squared_error,
+)
+
+__all__ = [
+    "mean_absolute_error",
+    "mean_squared_error",
+    "standardized_mean_squared_error",
+    "mean_standardized_log_loss",
+    "negative_log_predictive_density",
+    "quantile_coverage_error",
+]

qpytorch/mlls/__init__.py

@@ -0,0 +1,53 @@
+#!/usr/bin/env python3
+
+import warnings
+
+from gpytorch.mlls.added_loss_term import AddedLossTerm
+from .deep_approximate_mll import DeepApproximateMLL
+from .deep_predictive_log_likelihood import DeepPredictiveLogLikelihood
+from .exact_marginal_log_likelihood import ExactMarginalLogLikelihood
+from .gamma_robust_variational_elbo import GammaRobustVariationalELBO
+from .inducing_point_kernel_added_loss_term import InducingPointKernelAddedLossTerm
+from gpytorch.mlls.kl_gaussian_added_loss_term import KLGaussianAddedLossTerm
+from .kl_qexponential_added_loss_term import KLQExponentialAddedLossTerm
+from .leave_one_out_pseudo_likelihood import LeaveOneOutPseudoLikelihood
+from .marginal_log_likelihood import MarginalLogLikelihood
+from gpytorch.mlls.noise_model_added_loss_term import NoiseModelAddedLossTerm
+from .predictive_log_likelihood import PredictiveLogLikelihood
+from .sum_marginal_log_likelihood import SumMarginalLogLikelihood
+from .variational_elbo import VariationalELBO
+
+
+# Deprecated for 0.4 release
+class VariationalMarginalLogLikelihood(VariationalELBO):
+    def __init__(self, *args, **kwargs):
+        # Remove after 1.0
+        warnings.warn(
+            "VariationalMarginalLogLikelihood is deprecated. Please use VariationalELBO instead.", DeprecationWarning
+        )
+        super().__init__(*args, **kwargs)
+
+
+class VariationalELBOEmpirical(VariationalELBO):
+    def __init__(self, *args, **kwargs):
+        # Remove after 1.0
+        warnings.warn("VariationalELBOEmpirical is deprecated. Please use VariationalELBO instead.", DeprecationWarning)
+        super().__init__(*args, **kwargs)
+
+
+__all__ = [
+    "AddedLossTerm",
+    "DeepApproximateMLL",
+    "DeepPredictiveLogLikelihood",
+    "ExactMarginalLogLikelihood",
+    "InducingPointKernelAddedLossTerm",
+    "LeaveOneOutPseudoLikelihood",
+    "KLGaussianAddedLossTerm",
+    "KLQExponentialAddedLossTerm",
+    "MarginalLogLikelihood",
+    "NoiseModelAddedLossTerm",
+    "PredictiveLogLikelihood",
+    "GammaRobustVariationalELBO",
+    "SumMarginalLogLikelihood",
+    "VariationalELBO",
+]

qpytorch/mlls/_approximate_mll.py

@@ -0,0 +1,79 @@
+#!/usr/bin/env python3
+
+from abc import ABC, abstractmethod
+
+import torch
+
+from .marginal_log_likelihood import MarginalLogLikelihood
+
+
+class _ApproximateMarginalLogLikelihood(MarginalLogLikelihood, ABC):
+    r"""
+    An approximate marginal log likelihood (typically a bound) for approximate GP (QEP) models.
+    We expect that model is a :obj:`gpytorch.models.ApproximateGP` or :obj:`qpytorch.models.ApproximateQEP`.
+
+    Args:
+        likelihood (:obj:`qpytorch.likelihoods.Likelihood`):
+            The likelihood for the model
+        model (:obj:`gpytorch.models.ApproximateGP` or :obj:`qpytorch.models.ApproximateQEP`):
+            The approximate GP (QEP) model
+        num_data (int):
+            The total number of training data points (necessary for SGD)
+        beta (float - default 1.):
+            A multiplicative factor for the KL divergence term.
+            Setting it to 1 (default) recovers true variational inference
+            (as derived in `Scalable Variational Gaussian (Q-Exponential) Process Classification`_).
+            Setting it to anything less than 1 reduces the regularization effect of the model
+            (similarly to what was proposed in `the beta-VAE paper`_).
+        combine_terms (bool):
+            Whether or not to sum the expected NLL with the KL terms (default True)
+    """
+
+    def __init__(self, likelihood, model, num_data, beta=1.0, combine_terms=True):
+        super().__init__(likelihood, model)
+        self.combine_terms = combine_terms
+        self.num_data = num_data
+        self.beta = beta
+
+    @abstractmethod
+    def _log_likelihood_term(self, approximate_dist_f, target, **kwargs):
+        raise NotImplementedError
+
+    def forward(self, approximate_dist_f, target, **kwargs):
+        r"""
+        Computes the Variational ELBO given :math:`q(\mathbf f)` and `\mathbf y`.
+        Calling this function will call the likelihood's `expected_log_prob` function.
+
+        Args:
+            approximate_dist_f (:obj:`gpytorch.distributions.MultivariateNormal` or :obj:`qpytorch.distributions.MultivariateQExponential`):
+                :math:`q(\mathbf f)` the outputs of the latent function (the :obj:`gpytorch.models.ApproximateGP` or :obj:`qpytorch.models.ApproximateQEP`)
+            target (`torch.Tensor`):
+                :math:`\mathbf y` The target values
+
+        Keyword Args:
+            Additional arguments passed to the likelihood's `expected_log_prob` function.
+        """
+        # Get likelihood term and KL term
+        num_batch = approximate_dist_f.event_shape[0]
+        log_likelihood = self._log_likelihood_term(approximate_dist_f, target, **kwargs).div(num_batch)
+        kl_divergence = self.model.variational_strategy.kl_divergence().div(self.num_data / self.beta)
+
+        # Add any additional registered loss terms
+        added_loss = torch.zeros_like(log_likelihood)
+        had_added_losses = False
+        for added_loss_term in self.model.added_loss_terms():
+            added_loss.add_(added_loss_term.loss())
+            had_added_losses = True
+
+        # Log prior term
+        log_prior = torch.zeros_like(log_likelihood)
+        for name, module, prior, closure, _ in self.named_priors():
+            log_prior.add_(prior.log_prob(closure(module)).sum().div(self.num_data))
+
+        if self.combine_terms:
+            return log_likelihood - kl_divergence + log_prior - added_loss
+        else:
+            if had_added_losses:
+                return log_likelihood, kl_divergence, log_prior, added_loss
+            else:
+                return log_likelihood, kl_divergence, log_prior

qpytorch/mlls/deep_approximate_mll.py

@@ -0,0 +1,30 @@
+from ._approximate_mll import _ApproximateMarginalLogLikelihood
+
+
+class DeepApproximateMLL(_ApproximateMarginalLogLikelihood):
+    """
+    A wrapper to make a QPyTorch approximate marginal log likelihoods compatible with Deep QEPs.
+
+    Example:
+        >>> deep_mll = qpytorch.mlls.DeepApproximateMLL(
+        >>>     qpytorch.mlls.VariationalELBO(likelihood, model, num_data=1000)
+        >>> )
+
+    :param ~qpytorch.mlls._ApproximateMarginalLogLikelihood base_mll: The base
+        approximate MLL
+    """
+
+    def __init__(self, base_mll):
+        if not base_mll.combine_terms:
+            raise ValueError(
+                "The base marginal log likelihood object should combine terms "
+                "when used in conjunction with a DeepApproximateMLL."
+            )
+        super().__init__(base_mll.likelihood, base_mll.model, num_data=base_mll.num_data, beta=base_mll.beta)
+        self.base_mll = base_mll
+
+    def _log_likelihood_term(self, approximate_dist_f, target, **kwargs):
+        return self.base_mll._log_likelihood_term(approximate_dist_f, target, **kwargs).mean(0)
+
+    def forward(self, approximate_dist_f, target, **kwargs):
+        return self.base_mll.forward(approximate_dist_f, target, **kwargs).mean(0)

qpytorch/mlls/deep_predictive_log_likelihood.py

@@ -0,0 +1,32 @@
+from ..models.deep_qeps.dspp import DSPP
+from ._approximate_mll import _ApproximateMarginalLogLikelihood
+
+
+class DeepPredictiveLogLikelihood(_ApproximateMarginalLogLikelihood):
+    """
+    An implementation of the predictive log likelihood extended to DSPPs as discussed in Jankowiak et al., 2020.
+
+    If you are using a DSPP model, this is the loss object you want to create and optimize over.
+
+    This loss object is compatible only with models of type :obj:~qpytorch.models.deep_qeps.DSPP
+    """
+
+    def __init__(self, likelihood, model, num_data, beta=1.0, combine_terms=True):
+        if not combine_terms:
+            raise ValueError(
+                "The base marginal log likelihood object should combine terms "
+                "when used in conjunction with a DeepApproximateMLL."
+            )
+
+        if not isinstance(model, DSPP):
+            raise ValueError("DeepPredictiveLogLikelihood can only be used with a DSPP model.")
+
+        super().__init__(likelihood, model, num_data, beta, combine_terms)
+
+    def _log_likelihood_term(self, approximate_dist_f, target, **kwargs):
+        base_log_marginal = self.likelihood.log_marginal(target, approximate_dist_f, **kwargs)
+        deep_log_marginal = self.model.quad_weights.unsqueeze(-1) + base_log_marginal
+
+        deep_log_prob = deep_log_marginal.logsumexp(dim=0)
+
+        return deep_log_prob.sum(-1)

qpytorch/mlls/exact_marginal_log_likelihood.py

@@ -0,0 +1,96 @@
+#!/usr/bin/env python3
+
+from linear_operator.operators import MaskedLinearOperator
+
+from .. import settings
+from ..distributions import MultivariateNormal, MultivariateQExponential
+from ..likelihoods import _GaussianLikelihoodBase, _QExponentialLikelihoodBase
+from .marginal_log_likelihood import MarginalLogLikelihood
+
+
+class ExactMarginalLogLikelihood(MarginalLogLikelihood):
+    """
+    The exact marginal log likelihood (MLL) for an exact Gaussian (Q-Exponential) process with a
+    Gaussian (Q-Exponential) likelihood.
+
+    .. note::
+        This module will not work with anything other than a :obj:`~qpytorch.likelihoods.GaussianLikelihood`
+        (:obj:`~qpytorch.likelihoods.QExponentialLikelihood`) and a :obj:`~gpytorch.models.ExactGP` (:obj:`~qpytorch.models.ExactQEP`). 
+        It also cannot be used in conjunction with stochastic optimization.
+
+    :param ~qpytorch.likelihoods.GaussianLikelihood (~qpytorch.likelihoods.QExponentialLikelihood) likelihood: The Gaussian (Q-Exponential) likelihood for the model
+    :param ~gpytorch.models.ExactGP (~qpytorch.models.ExactQEP) model: The exact GP (QEP) model
+
+    Example:
+        >>> # model is a qpytorch.models.ExactGP or qpytorch.models.ExactQEP
+        >>> # likelihood is a qpytorch.likelihoods.Likelihood
+        >>> mll = qpytorch.mlls.ExactMarginalLogLikelihood(likelihood, model)
+        >>>
+        >>> output = model(train_x)
+        >>> loss = -mll(output, train_y)
+        >>> loss.backward()
+    """
+
+    def __init__(self, likelihood, model):
+        if not isinstance(likelihood, (_GaussianLikelihoodBase, _QExponentialLikelihoodBase)):
+            raise RuntimeError("Likelihood must be Gaussian or Q-Exponential for exact inference")
+        super(ExactMarginalLogLikelihood, self).__init__(likelihood, model)
+
+    def _add_other_terms(self, res, params):
+        # Add additional terms (SGPR / learned inducing points, heteroskedastic likelihood models)
+        for added_loss_term in self.model.added_loss_terms():
+            res = res.add(added_loss_term.loss(*params))
+
+        # Add log probs of priors on the (functions of) parameters
+        res_ndim = res.ndim
+        for name, module, prior, closure, _ in self.model.named_priors():
+            prior_term = prior.log_prob(closure(module))
+            res.add_(prior_term.view(*prior_term.shape[:res_ndim], -1).sum(dim=-1))
+
+        return res
+
+    def forward(self, function_dist, target, *params, **kwargs):
+        r"""
+        Computes the MLL given :math:`p(\mathbf f)` and :math:`\mathbf y`.
+
+        :param ~gpytorch.distributions.MultivariateNormal or ~qpytorch.distributions.MultivariateQExponential function_dist: :math:`p(\mathbf f)`
+            the outputs of the latent function (the :obj:`gpytorch.models.ExactGP` or :obj:`qpytorch.models.ExactQEP`)
+        :param torch.Tensor target: :math:`\mathbf y` The target values
+        :rtype: torch.Tensor
+        :return: Exact MLL. Output shape corresponds to batch shape of the model/input data.
+        """
+        if not isinstance(function_dist, (MultivariateNormal, MultivariateQExponential)):
+            raise RuntimeError("ExactMarginalLogLikelihood can only operate on Gaussian or Q-Exponential random variables")
+
+        # Determine output likelihood
+        output = self.likelihood(function_dist, *params, **kwargs)
+
+        # Remove NaN values if enabled
+        if settings.observation_nan_policy.value() == "mask":
+            observed = settings.observation_nan_policy._get_observed(target, output.event_shape)
+            if isinstance(function_dist, MultivariateNormal):
+                output = MultivariateNormal(
+                    mean=output.mean[..., observed],
+                    covariance_matrix=MaskedLinearOperator(
+                        output.lazy_covariance_matrix, observed.reshape(-1), observed.reshape(-1)
+                    ),
+                )
+            elif isinstance(function_dist, MultivariateQExponential):
+                output = MultivariateQExponential(
+                    mean=output.mean[..., observed],
+                    covariance_matrix=MaskedLinearOperator(
+                        output.lazy_covariance_matrix, observed.reshape(-1), observed.reshape(-1)
+                    ),
+                    power=output.power
+                )
+            target = target[..., observed]
+        elif settings.observation_nan_policy.value() == "fill":
+            raise ValueError("NaN observation policy 'fill' is not supported by ExactMarginalLogLikelihood!")
+
+        # Get the log prob of the marginal distribution
+        res = output.log_prob(target)
+        res = self._add_other_terms(res, params)
+
+        # Scale by the amount of data we have
+        num_data = function_dist.event_shape.numel()
+        return res.div_(num_data)

qpytorch/mlls/gamma_robust_variational_elbo.py

@@ -0,0 +1,106 @@
+#!/usr/bin/env python3
+
+import math
+
+import numpy as np
+import torch
+
+from ..likelihoods import _GaussianLikelihoodBase, _QExponentialLikelihoodBase
+from ._approximate_mll import _ApproximateMarginalLogLikelihood
+
+
+class GammaRobustVariationalELBO(_ApproximateMarginalLogLikelihood):
+    r"""
+    An alternative to the variational evidence lower bound (ELBO), proposed by `Knoblauch, 2019`_.
+    It is derived by replacing the log-likelihood term in the ELBO with a `\gamma` divergence:
+
+    .. math::
+
+       \begin{align*}
+          \mathcal{L}_{\gamma} &=
+          \sum_{i=1}^N \mathbb{E}_{q( \mathbf u)} \left[
+            -\frac{\gamma}{\gamma - 1}
+            \frac{
+                p( y_i \! \mid \! \mathbf u, x_i)^{\gamma - 1}
+            }{
+                \int p(y \mid \mathbf u, x_i)^{\gamma} \: dy
+            }
+          \right] - \beta \: \text{KL} \left[ q( \mathbf u) \Vert p( \mathbf u) \right]
+       \end{align*}
+
+    where :math:`N` is the number of datapoints, :math:`\gamma` is a hyperparameter,
+    :math:`q(\mathbf u)` is the variational distribution for
+    the inducing function values, and :math:`p(\mathbf u)` is the prior distribution for the inducing function
+    values.
+
+    :math:`\beta` is a scaling constant for the KL divergence.
+
+    .. note::
+        This module will only work with :obj:`~qpytorch.likelihoods.GaussianLikelihood` or :obj:`~qpytorch.likelihoods.QExponentialLikelihood`.
+
+    :param ~qpytorch.likelihoods.GaussianLikelihood (~qpytorch.likelihoods.QExponentialLikelihood) likelihood: The likelihood for the model
+    :param ~gpytorch.models.ApproximateGP (~qpytorch.models.ApproximateQEP) model: The approximate GP (QEP) model
+    :param int num_data: The total number of training data points (necessary for SGD)
+    :param float beta: (optional, default=1.) A multiplicative factor for the KL divergence term.
+        Setting it to anything less than 1 reduces the regularization effect of the model
+        (similarly to what was proposed in `the beta-VAE paper`_).
+    :param float gamma: (optional, default=1.03) The :math:`\gamma`-divergence hyperparameter.
+    :param bool combine_terms: (default=True): Whether or not to sum the
+        expected NLL with the KL terms (default True)
+
+    Example:
+        >>> # model is a qpytorch.models.ApproximateGP or qpytorch.models.ApproximateQEP
+        >>> # likelihood is a qpytorch.likelihoods.Likelihood
+        >>> mll = qpytorch.mlls.GammaRobustVariationalELBO(likelihood, model, num_data=100, beta=0.5, gamma=1.03)
+        >>>
+        >>> output = model(train_x)
+        >>> loss = -mll(output, train_y)
+        >>> loss.backward()
+
+    .. _Knoblauch, 2019:
+        https://arxiv.org/pdf/1904.02303.pdf
+    .. _Knoblauch, Jewson, Damoulas 2019:
+        https://arxiv.org/pdf/1904.02063.pdf
+    """
+
+    def __init__(self, likelihood, model, gamma=1.03, *args, **kwargs):
+        if not isinstance(likelihood, (_GaussianLikelihoodBase, _QExponentialLikelihoodBase)):
+            raise RuntimeError("Likelihood must be Gaussian or Q-Exponential for exact inference")
+        super().__init__(likelihood, model, *args, **kwargs)
+        if gamma <= 1.0:
+            raise ValueError("gamma should be > 1.0")
+        self.gamma = gamma
+
+    def _log_likelihood_term(self, variational_dist_f, target, *args, **kwargs):
+        shifted_gamma = self.gamma - 1
+
+        muf, varf = variational_dist_f.mean, variational_dist_f.variance
+
+        # Get noise from likelihood
+        noise = self.likelihood._shaped_noise_covar(muf.shape, *args, **kwargs).diagonal(dim1=-1, dim2=-2)
+        # Potentially reshape the noise to deal with the multitask case
+        noise = noise.view(*noise.shape[:-1], *variational_dist_f.event_shape)
+
+        # adapted from https://github.com/JeremiasKnoblauch/GVIPublic/
+        mut = shifted_gamma * target / noise + muf / varf
+        sigmat = 1.0 / (shifted_gamma / noise + 1.0 / varf)
+        log_integral = -0.5 * shifted_gamma * torch.log(2.0 * math.pi * noise) - 0.5 * np.log1p(shifted_gamma)
+        log_tempered = (
+            -math.log(shifted_gamma)
+            - 0.5 * shifted_gamma * torch.log(2.0 * math.pi * noise)
+            - 0.5 * torch.log1p(shifted_gamma * varf / noise)
+            - 0.5 * (shifted_gamma * target.pow(2.0) / noise)
+            - 0.5 * muf.pow(2.0) / varf
+            + 0.5 * mut.pow(2.0) * sigmat
+        )
+        # TODO: verify for Q-Exponential
+
+        factor = log_tempered + shifted_gamma / self.gamma * log_integral
+        factor = self.gamma * factor.exp()
+
+        # Do appropriate summation for multitask Gaussian (Q-Exponential) likelihoods
+        num_event_dim = len(variational_dist_f.event_shape)
+        if num_event_dim > 1:
+            factor = factor.sum(list(range(-1, -num_event_dim, -1)))
+
+        return factor.sum(-1)