gpflow 2.8.0__tar.gz → 2.9.0__tar.gz

{gpflow-2.8.0/gpflow.egg-info → gpflow-2.9.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: gpflow
-Version: 2.8.0
+Version: 2.9.0
 Summary: Gaussian process methods in TensorFlow
 Home-page: https://www.gpflow.org
 Author: James Hensman, Alex Matthews

{gpflow-2.8.0 → gpflow-2.9.0}/RELEASE.md

@@ -33,7 +33,7 @@ This release contains contributions from:
 <INSERT>, <NAME>, <HERE>, <USING>, <GITHUB>, <HANDLE>
 
 
-# Release 2.9.0 (next release)
+# Release 2.10.0 (next release)
 
 <INSERT SMALL BLURB ABOUT RELEASE FOCUS AREA AND POTENTIAL TOOLCHAIN CHANGES>
 
@@ -66,6 +66,44 @@ This release contains contributions from:
 <INSERT>, <NAME>, <HERE>, <USING>, <GITHUB>, <HANDLE>
 
 
+# Release 2.9.0
+
+This release adds caching of compiled graphs inside the Scipy optimizer, and adds support
+for returning loss history. It also adds supports for Python 3.11.
+
+## Major Features and Improvements
+
+* Support returning loss history with Scipy optimizer. 
+* Scipy minimize wrapper caches compiled graphs and re-uses them if called with the same arguments.
+  This functionality can be disabled by setting the new `compile_cache_size` argument to 0.
+
+## Bug Fixes and Other Changes
+
+* Support and test with Python 3.11
+* Test against a 'production' environment (in addition to 'min' and 'max' environments).
+
+## Thanks to our Contributors
+
+This release contains contributions from:
+
+khurram-ghani, jesnie, uri-granta
+
+
+# Release 2.8.1
+
+A small fix to ensure support for (and testing with) TensorFlow 2.12.
+
+## Bug Fixes and Other Changes
+
+* Support and test with TensorFlow 2.12
+
+## Thanks to our Contributors
+
+This release contains contributions from:
+
+uri-granta
+
+
 # Release 2.8.0
 
 The main focus of this release is to provide users control over arguments for `tf.function`

gpflow-2.9.0/VERSION

@@ -0,0 +1 @@
+2.9.0

{gpflow-2.8.0 → gpflow-2.9.0}/doc/sphinx/conf.py

@@ -24,7 +24,7 @@ copyright = "2023, The GPflow Contributors"
 author = "The GPflow Contributors"
 
 # The full version, including alpha/beta/rc tags
-release = "2.8.0"
+release = "2.9.0"
 
 # -- General configuration ---------------------------------------------------
 

{gpflow-2.8.0 → gpflow-2.9.0}/doc/sphinx/installation.rst

@@ -25,6 +25,10 @@ explicitly install specific versions of these.
 +---------------------+---------------------------------+
 | 2.10.*              | 0.18.*                          |
 +---------------------+---------------------------------+
+| 2.11.*              | 0.19.*                          |
++---------------------+---------------------------------+
+| 2.12.*              | 0.20.*                          |
++---------------------+---------------------------------+
 
 Second, a word of warning about new Mac computers. On new Mac machines you will need to install
 ``tensorflow-macos`` instead of the regular ``tensorflow``.

{gpflow-2.8.0 → gpflow-2.9.0}/gpflow/config/__config__.py

@@ -50,10 +50,9 @@ manager :func:`as_context`:
 """
 
 import contextlib
-import enum
 import os
 from dataclasses import dataclass, field, replace
-from typing import Any, Dict, Generator, List, Mapping, Optional, Tuple, Union
+from typing import Any, Dict, Generator, List, Mapping, Optional, Union
 
 import numpy as np
 import tabulate

{gpflow-2.8.0 → gpflow-2.9.0}/gpflow/kernels/categorical.py

@@ -16,7 +16,6 @@ from typing import Any, Optional
 
 import numpy as np
 import tensorflow as tf
-import tensorflow_probability as tfp
 
 from .. import default_int, set_trainable
 from ..base import Parameter, TensorType

{gpflow-2.8.0 → gpflow-2.9.0}/gpflow/optimizers/natgrad.py

@@ -201,6 +201,8 @@ class NaturalGradient(tf.optimizers.Optimizer):
         """
         name = self.__class__.__name__ if name is None else name
         super().__init__(name)
+        # explicitly store name (as TF <2.12 stores it differently from TF 2.12+)
+        self.natgrad_name = name
         self.gamma = gamma
         self.xi_transform = xi_transform
 
@@ -263,7 +265,7 @@ class NaturalGradient(tf.optimizers.Optimizer):
         q_mu_grads, q_sqrt_grads = tape.gradient(loss, [q_mu_vars, q_sqrt_vars])
         # NOTE that these are the gradients in *unconstrained* space
 
-        with tf.name_scope(f"{self._name}/natural_gradient_steps"):
+        with tf.name_scope(f"{self.natgrad_name}/natural_gradient_steps"):
             for q_mu_grad, q_sqrt_grad, q_mu, q_sqrt, xi_transform in zip(
                 q_mu_grads, q_sqrt_grads, q_mus, q_sqrts, xis
             ):

{gpflow-2.8.0 → gpflow-2.9.0}/gpflow/optimizers/scipy.py

@@ -13,7 +13,19 @@
 # limitations under the License.
 
 import warnings
-from typing import Any, Callable, Iterable, List, Mapping, Optional, Sequence, Tuple, Union
+from collections import OrderedDict
+from typing import (
+    Any,
+    Callable,
+    FrozenSet,
+    Iterable,
+    List,
+    Mapping,
+    Optional,
+    Sequence,
+    Tuple,
+    Union,
+)
 
 import numpy as np
 import scipy.optimize
@@ -31,6 +43,31 @@ LossClosure = Callable[[], tf.Tensor]
 
 
 class Scipy:
+    def __init__(self, compile_cache_size: int = 2) -> None:
+        """
+        Wrapper around the scipy optimizer.
+
+        :param compile_cache_size: The number of compiled evalutation functions to cache for calls
+            to `minimize`. Only applies when `compile` argument to `minimize` is True.
+
+            The compiled evaluation functions are cached so that subsequent calls to `minimize` with
+            the same `closure`, `variables`, `allow_unused_variables`, and `tf_fun_args` will reuse
+            a previously compiled function. Up to `compile_cache_size` most recent functions are
+            cached. This can be disabled by setting `compile_cache_size` to 0.
+        """
+        self.compile_cache: OrderedDict[
+            Tuple[Callable[[], Any], Tuple[int, ...], FrozenSet[Tuple[str, Any]], bool],
+            tf.function,
+        ] = OrderedDict()
+
+        if compile_cache_size < 0:
+            raise ValueError(
+                "The 'compile_cache_size' argument must be non-negative, got {}.".format(
+                    compile_cache_size
+                )
+            )
+        self.compile_cache_size = compile_cache_size
+
     def minimize(
         self,
         closure: LossClosure,
@@ -40,6 +77,7 @@ class Scipy:
         compile: bool = True,
         allow_unused_variables: bool = False,
         tf_fun_args: Optional[Mapping[str, Any]] = None,
+        track_loss_history: bool = False,
         **scipy_kwargs: Any,
     ) -> OptimizeResult:
         """
@@ -68,6 +106,8 @@ class Scipy:
 
                 opt = gpflow.optimizers.Scipy()
                 opt.minimize(..., compile=True, tf_fun_args=dict(jit_compile=True))
+        :param track_loss_history: Whether to track the training loss history and return it in
+            the optimization result.
         :param scipy_kwargs: Arguments passed through to `scipy.optimize.minimize`.
             Note that Scipy's minimize() takes a `callback` argument, but you probably want to use
             our wrapper and pass in `step_callback`.
@@ -98,16 +138,24 @@ class Scipy:
             allow_unused_variables=allow_unused_variables,
             tf_fun_args=tf_fun_args,
         )
+
         if step_callback is not None:
             if "callback" in scipy_kwargs:
                 raise ValueError("Callback passed both via `step_callback` and `callback`")
-
             callback = self.callback_func(variables, step_callback)
-            scipy_kwargs.update(dict(callback=callback))
+            scipy_kwargs["callback"] = callback
+        history: List[AnyNDArray] = []
+        if track_loss_history:
+            callback = self.loss_history_callback_func(func, history, scipy_kwargs.get("callback"))
+            scipy_kwargs["callback"] = callback
 
         opt_result = scipy.optimize.minimize(
             func, initial_params, jac=True, method=method, **scipy_kwargs
         )
+
+        if track_loss_history:
+            opt_result["loss_history"] = history
+
         values = self.unpack_tensors(variables, opt_result.x)
         self.assign_tensors(variables, values)
         return opt_result
@@ -116,9 +164,8 @@ class Scipy:
     def initial_parameters(cls, variables: Sequence[tf.Variable]) -> tf.Tensor:
         return cls.pack_tensors(variables)
 
-    @classmethod
     def eval_func(
-        cls,
+        self,
         closure: LossClosure,
         variables: Sequence[tf.Variable],
         tf_fun_args: Mapping[str, Any],
@@ -130,25 +177,41 @@ class Scipy:
         def _tf_eval(x: tf.Tensor) -> Tuple[tf.Tensor, tf.Tensor]:
             nonlocal first_call
 
-            values = cls.unpack_tensors(variables, x)
-            cls.assign_tensors(variables, values)
+            values = self.unpack_tensors(variables, x)
+            self.assign_tensors(variables, values)
 
             if first_call:
                 # Only check for unconnected gradients on the first function evaluation.
                 loss, grads = _compute_loss_and_gradients(
                     closure, variables, tf.UnconnectedGradients.NONE
                 )
-                grads = cls._filter_unused_variables(variables, grads, allow_unused_variables)
+                grads = self._filter_unused_variables(variables, grads, allow_unused_variables)
                 first_call = False
             else:
                 loss, grads = _compute_loss_and_gradients(
                     closure, variables, tf.UnconnectedGradients.ZERO
                 )
 
-            return loss, cls.pack_tensors(grads)
+            return loss, self.pack_tensors(grads)
 
         if compile:
-            _tf_eval = tf.function(_tf_eval, **tf_fun_args)
+            # Re-use the same tf.function graph for calls to minimize, as long as the arguments
+            # affecting the graph are the same. This can boost performance of use cases where
+            # minimize is called repeatedly with the same model loss.
+            key = (
+                closure,
+                tuple(id(v) for v in variables),
+                frozenset(tf_fun_args.items()),
+                allow_unused_variables,
+            )
+            if self.compile_cache_size > 0:
+                if key not in self.compile_cache:
+                    if len(self.compile_cache) >= self.compile_cache_size:
+                        self.compile_cache.popitem(last=False)  # Remove the oldest entry.
+                    self.compile_cache[key] = tf.function(_tf_eval, **tf_fun_args)
+                _tf_eval = self.compile_cache[key]
+            else:
+                _tf_eval = tf.function(_tf_eval, **tf_fun_args)
 
         def _eval(x: AnyNDArray) -> Tuple[AnyNDArray, AnyNDArray]:
             loss, grad = _tf_eval(tf.convert_to_tensor(x))
@@ -186,7 +249,8 @@ class Scipy:
     def callback_func(
         cls, variables: Sequence[tf.Variable], step_callback: StepCallback
     ) -> Callable[[AnyNDArray], None]:
-        step = 0  # type: int
+        # Convert a step_callback function to a Scipy callback function
+        step: int = 0
 
         def _callback(x: AnyNDArray) -> None:
             nonlocal step
@@ -201,6 +265,23 @@ class Scipy:
 
         return _callback
 
+    @classmethod
+    def loss_history_callback_func(
+        cls,
+        minimize_func: Callable[[AnyNDArray], Tuple[AnyNDArray, AnyNDArray]],
+        history: List[AnyNDArray],
+        callback: Optional[Callable[[AnyNDArray], None]] = None,
+    ) -> Callable[[AnyNDArray], None]:
+        # Return a Scipy callback function that tracks loss history, optionally combined
+        # with another callback.
+
+        def _callback(x: AnyNDArray) -> None:
+            if callback is not None:
+                callback(x)
+            history.append(minimize_func(x)[0])
+
+        return _callback
+
     @staticmethod
     def pack_tensors(tensors: Sequence[Union[tf.Tensor, tf.Variable]]) -> tf.Tensor:
         flats = [tf.reshape(tensor, (-1,)) for tensor in tensors]

{gpflow-2.8.0 → gpflow-2.9.0/gpflow.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: gpflow
-Version: 2.8.0
+Version: 2.9.0
 Summary: Gaussian process methods in TensorFlow
 Home-page: https://www.gpflow.org
 Author: James Hensman, Alex Matthews

{gpflow-2.8.0 → gpflow-2.9.0}/tests/gpflow/expectations/test_expectations.py

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from typing import Any, Callable, Iterable, Mapping, Sequence
+from typing import Any, Callable, Iterable, Mapping, Optional, Sequence
 
 import numpy as np
 import pytest
@@ -128,9 +128,9 @@ def inducing_variable() -> iv.InducingVariables:
     return iv.InducingPoints(Z)
 
 
-def _check(params: Iterable[Any]) -> None:
+def _check(params: Iterable[Any], nghp: Optional[int] = None) -> None:
     analytic = expectation(*params)
-    quad = quadrature_expectation(*params)
+    quad = quadrature_expectation(*params, nghp=nghp)  # type: ignore[misc]
     assert_allclose(analytic, quad, rtol=RTOL)
 
 
@@ -302,7 +302,7 @@ def test_exKxz_markov(
     mean: mf.MeanFunction,
     inducing_variable: iv.InducingVariables,
 ) -> None:
-    _check((distribution, (kernel, inducing_variable), mean))
+    _check((distribution, (kernel, inducing_variable), mean), nghp=20)
 
 
 @pytest.mark.parametrize("distribution", distrs("dirac_markov_gauss"))

{gpflow-2.8.0 → gpflow-2.9.0}/tests/gpflow/models/test_methods.py

@@ -11,8 +11,6 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
-from dataclasses import dataclass
 from typing import List
 
 import numpy as np
@@ -27,7 +25,6 @@ from gpflow.base import AnyNDArray
 # ------------------------------------------
 
 
-@dataclass(frozen=True)
 class Datum:
     rng: np.random.RandomState = np.random.RandomState(0)
     X: AnyNDArray = rng.randn(100, 2)

{gpflow-2.8.0 → gpflow-2.9.0}/tests/gpflow/models/test_sgpr.py

@@ -11,8 +11,6 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-from dataclasses import dataclass
-
 import numpy as np
 import tensorflow as tf
 
@@ -21,7 +19,6 @@ from gpflow.base import AnyNDArray
 from gpflow.utilities import to_default_float
 
 
-@dataclass(frozen=True)
 class Datum:
     rng: np.random.RandomState = np.random.RandomState(0)
     X: AnyNDArray = rng.randn(100, 2)

{gpflow-2.8.0 → gpflow-2.9.0}/tests/gpflow/models/test_svgp.py

@@ -11,8 +11,6 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
-from dataclasses import dataclass
 from typing import Sequence
 
 import numpy as np
@@ -26,7 +24,6 @@ from gpflow.base import AnyNDArray
 from gpflow.models import SVGP
 
 
-@dataclass(frozen=True)
 class DatumSVGP:
     rng: np.random.RandomState = np.random.RandomState(0)
     X: AnyNDArray = rng.randn(20, 1)