mlpp-lib 0.5.2__tar.gz → 0.7.0__tar.gz

{mlpp_lib-0.5.2 → mlpp_lib-0.7.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: mlpp-lib
-Version: 0.5.2
+Version: 0.7.0
 Summary: Collection of methods for ML-based postprocessing of weather forecasts.
 Author: Daniele Nerini
 Author-email: daniele.nerini@meteoswiss.ch
@@ -17,8 +17,8 @@ Requires-Dist: numpy (>=1.22.1,<2.0.0)
 Requires-Dist: pandas (>=1.4.1,<2.0.0)
 Requires-Dist: properscoring (>=0.1,<0.2)
 Requires-Dist: scipy (>=1.8.0,<2.0.0)
-Requires-Dist: tensorflow (>=2.9.3,<2.10.0) ; extra == "tensorflow" or extra == "extended"
-Requires-Dist: tensorflow-cpu (>=2.9.3,<2.10.0) ; extra == "tensorflow-cpu"
-Requires-Dist: tensorflow-probability (>=0.17.0,<0.18.0)
+Requires-Dist: tensorflow (>=2.12.1,<2.13.0) ; extra == "tensorflow" or extra == "extended"
+Requires-Dist: tensorflow-cpu (>=2.12.1,<2.13.0) ; extra == "tensorflow-cpu"
+Requires-Dist: tensorflow-probability (>=0.20.0,<0.21.0)
 Requires-Dist: xarray (>=2022.3.0,<2023.0.0)
 Requires-Dist: zarr (>=2.14.1,<3.0.0) ; extra == "extended"

{mlpp_lib-0.5.2 → mlpp_lib-0.7.0}/README.md

@@ -20,15 +20,22 @@ import pandas as pd
 from mlpp_lib.datasets import DataModule, DataSplitter
 ```
 
-<details>
-<summary> click here to visualize </summary>
+    2024-03-12 11:01:48.532698: I tensorflow/tsl/cuda/cudart_stub.cc:28] Could not find cuda drivers on your machine, GPU will not be used.
+    2024-03-12 11:01:48.594233: I tensorflow/tsl/cuda/cudart_stub.cc:28] Could not find cuda drivers on your machine, GPU will not be used.
+    2024-03-12 11:01:48.595154: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.
+    To enable the following instructions: AVX2 AVX512F FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.
+
+
+    2024-03-12 11:01:49.442240: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT
+
+
 
 ```python
 LEADTIMES = np.arange(24)
 REFTIMES = pd.date_range("2018-01-01", "2018-03-31", freq="24H")
 STATIONS = [chr(i) * 3 for i in range(ord("A"), ord("Z"))]
 SHAPE = (len(REFTIMES), len(LEADTIMES), len(STATIONS))
-DIMS = ["forecast_reference_time", "t", "station"]
+DIMS = ["forecast_reference_time", "lead_time", "station"]
 
 def features_dataset() -> xr.Dataset:
     rng = np.random.default_rng(1)
@@ -44,7 +51,7 @@ def features_dataset() -> xr.Dataset:
         },
         coords={
             "forecast_reference_time": REFTIMES,
-            "t": LEADTIMES,
+            "lead_time": LEADTIMES,
             "station": STATIONS,
         },
     )
@@ -63,7 +70,7 @@ def targets_dataset() -> xr.Dataset:
         {"obs:y1": (DIMS, Y[..., 0]), "obs:y2": (DIMS, Y[..., 1])},
         coords={
             "forecast_reference_time": REFTIMES,
-            "t": LEADTIMES,
+            "lead_time": LEADTIMES,
             "station": STATIONS,
         },
     )
@@ -72,9 +79,7 @@ def targets_dataset() -> xr.Dataset:
 
 
 ```
-</details>
 
-## Raw datasets
 MLPP expects xarray objects that look like this:
 
 
@@ -85,16 +90,17 @@ print(features)
 ```
 
     <xarray.Dataset>
-    Dimensions:                  (forecast_reference_time: 90, t: 24, station: 25)
+    Dimensions:                  (forecast_reference_time: 90, lead_time: 24,
+                                  station: 25)
     Coordinates:
       * forecast_reference_time  (forecast_reference_time) datetime64[ns] 2018-01...
-      * t                        (t) int64 0 1 2 3 4 5 6 7 ... 17 18 19 20 21 22 23
+      * lead_time                (lead_time) int64 0 1 2 3 4 5 ... 18 19 20 21 22 23
       * station                  (station) <U3 'AAA' 'BBB' 'CCC' ... 'XXX' 'YYY'
     Data variables:
-        coe:x1                   (forecast_reference_time, t, station) float64 0....
-        coe:x2                   (forecast_reference_time, t, station) float64 0....
-        obs:x3                   (forecast_reference_time, t, station) float64 0....
-        dem:x4                   (forecast_reference_time, t, station) float64 -1...
+        coe:x1                   (forecast_reference_time, lead_time, station) float64 ...
+        coe:x2                   (forecast_reference_time, lead_time, station) float64 ...
+        obs:x3                   (forecast_reference_time, lead_time, station) float64 ...
+        dem:x4                   (forecast_reference_time, lead_time, station) float64 ...
 
 
 
@@ -104,14 +110,15 @@ print(targets)
 ```
 
     <xarray.Dataset>
-    Dimensions:                  (forecast_reference_time: 90, t: 24, station: 25)
+    Dimensions:                  (forecast_reference_time: 90, lead_time: 24,
+                                  station: 25)
     Coordinates:
       * forecast_reference_time  (forecast_reference_time) datetime64[ns] 2018-01...
-      * t                        (t) int64 0 1 2 3 4 5 6 7 ... 17 18 19 20 21 22 23
+      * lead_time                (lead_time) int64 0 1 2 3 4 5 ... 18 19 20 21 22 23
       * station                  (station) <U3 'AAA' 'BBB' 'CCC' ... 'XXX' 'YYY'
     Data variables:
-        obs:y1                   (forecast_reference_time, t, station) float64 0....
-        obs:y2                   (forecast_reference_time, t, station) float64 0....
+        obs:y1                   (forecast_reference_time, lead_time, station) float64 ...
+        obs:y2                   (forecast_reference_time, lead_time, station) float64 ...
 
 
 ## Preparing data
@@ -133,7 +140,7 @@ splitter = DataSplitter(
 
 datamodule = DataModule(
     features, targets[["obs:y1"]],
-    batch_dims=["forecast_reference_time","t","station"],
+    batch_dims=["forecast_reference_time", "lead_time", "station"],
     splitter=splitter
 )
 
@@ -168,9 +175,10 @@ history = model.fit(
 ```
 
     Epoch 1/2
-    689/689 [==============================] - 2s 2ms/step - loss: 0.5629 - val_loss: 0.5716
+    689/689 [==============================] - 2s 2ms/step - loss: 0.5633 - val_loss: 0.5721
+
     Epoch 2/2
-    689/689 [==============================] - 1s 2ms/step - loss: 0.5601 - val_loss: 0.5709
+    689/689 [==============================] - 1s 2ms/step - loss: 0.5607 - val_loss: 0.5695
 
 
 ## Predictions
@@ -185,12 +193,11 @@ print(test_pred_ensemble)
 
     <xarray.Dataset>
     Dimensions:                  (realization: 21, forecast_reference_time: 18,
-                                  t: 24, station: 5)
+                                  lead_time: 24, station: 5)
     Coordinates:
       * forecast_reference_time  (forecast_reference_time) datetime64[ns] 2018-03...
-      * t                        (t) int64 0 1 2 3 4 5 6 7 ... 17 18 19 20 21 22 23
+      * lead_time                (lead_time) int64 0 1 2 3 4 5 ... 18 19 20 21 22 23
       * station                  (station) <U3 'AAA' 'EEE' 'JJJ' 'PPP' 'RRR'
       * realization              (realization) int64 0 1 2 3 4 5 ... 16 17 18 19 20
     Data variables:
-        obs:y1                   (realization, forecast_reference_time, t, station) float64 ...
-
+        obs:y1                   (realization, forecast_reference_time, lead_time, station) float64 ...

{mlpp_lib-0.5.2 → mlpp_lib-0.7.0}/mlpp_lib/losses.py

@@ -576,3 +576,101 @@ class MultivariateLoss(tf.keras.losses.Loss):
         y_pred.shape = (*y_pred.batch_shape, *y_pred.event_shape)
 
         return y_true, y_pred
+
+
+class BinaryClassifierLoss(tf.keras.losses.Loss):
+    """
+    Compute binary classification loss from continuous predictions based on a threshold.
+
+    Parameters
+    ----------
+    threshold: float
+    loss_type: {"binary_crossentropy", "focal"}
+        The type of loss to be used.
+    n_samples: int, optional
+    **kwargs:
+        (Optional) Additional keyword arguments to be passed to the parent `Loss` class.
+
+    """
+
+    def __init__(
+        self,
+        threshold: float,
+        loss_type: Literal["binary_crossentropy", "focal"] = "binary_crossentropy",
+        n_samples: int = 1000,
+        **kwargs,
+    ) -> None:
+        super(BinaryClassifierLoss, self).__init__(**kwargs)
+
+        self.threshold = float(threshold)
+        self.n_samples = int(n_samples)
+        if self.n_samples < 2:
+            raise ValueError("n_samples must be > 1")
+        self.loss_type = loss_type
+
+    def get_config(self) -> dict:
+        custom_config = {
+            "threshold": self.threshold,
+            "loss_type": self.loss_type,
+            "n_samples": self.n_samples,
+        }
+        config = super().get_config()
+        config.update(custom_config)
+        return config
+
+    def call(
+        self,
+        y_true: Union[tf.Tensor, np.ndarray],
+        y_pred: Union[tf.Tensor, np.ndarray, tfp.distributions.Distribution],
+    ) -> tf.Tensor:
+        """
+        Compute the loss.
+
+        Parameters
+        ----------
+        y_true: array-like
+            Values representing the ground truth.
+        y_pred: array_like or tfp.Distribution
+            Predicted values or distributions.
+        """
+        threshold = tf.constant(self.threshold, dtype=y_true.dtype)
+        n_samples = self.n_samples
+        y_true = tf.debugging.check_numerics(y_true, "Target values")
+
+        if isinstance(y_pred, tfp.distributions.Distribution):
+            y_pred_samples = y_pred.sample(n_samples)
+        else:
+            y_pred_samples = y_pred
+
+        y_pred_samples = tf.debugging.check_numerics(y_pred_samples, "Predicted values")
+
+        y_true_bool = tf.cast(y_true > threshold, dtype=y_true.dtype)
+        y_pred_bool = tf.cast(y_pred_samples > threshold, dtype=y_true.dtype)
+        y_pred_prob = tf.reduce_mean(y_pred_bool, axis=0)
+
+        loss = tf.keras.losses.binary_crossentropy(y_true_bool, y_pred_prob, axis=1)
+        if self.loss_type == "focal":
+            loss = tf.pow(1 - tf.exp(-loss), 2)
+
+        return loss
+
+
+class CombinedLoss(tf.keras.losses.Loss):
+    def __init__(self, losses):
+        # Local import to avoid circular dependency with mlpp_lib.utils
+        from mlpp_lib.utils import get_loss
+
+        super(CombinedLoss, self).__init__()
+        self.losses = []
+        self.weights = []
+
+        # Initialize losses based on the input config dictionaries
+        for loss_config in losses:
+            self.weights.append(loss_config.get("weight", 1.0))
+            self.losses.append(get_loss(loss_config))
+
+    def call(self, y_true, y_pred):
+        total_loss = 0
+        for loss, weight in zip(self.losses, self.weights):
+            total_loss += weight * loss(y_true, y_pred)
+        return total_loss

{mlpp_lib-0.5.2 → mlpp_lib-0.7.0}/mlpp_lib/models.py

@@ -26,7 +26,7 @@ else:
 _LOGGER = logging.getLogger(__name__)
 
 
-@tf.keras.utils.register_keras_serializable()
+@tf.keras.saving.register_keras_serializable()
 class MonteCarloDropout(Dropout):
     def call(self, inputs):
         return super().call(inputs, training=True)

{mlpp_lib-0.5.2 → mlpp_lib-0.7.0}/mlpp_lib/probabilistic_layers.py

@@ -22,7 +22,7 @@ from tensorflow_probability.python.layers import (
 )
 
 
-@tf.keras.utils.register_keras_serializable()
+@tf.keras.saving.register_keras_serializable()
 class IndependentBeta(tfpl.DistributionLambda):
     """An independent 4-parameter Beta Keras layer."""
 
@@ -135,7 +135,7 @@ class IndependentBeta(tfpl.DistributionLambda):
         return super(IndependentBeta, self).output[0]
 
 
-@tf.keras.utils.register_keras_serializable()
+@tf.keras.saving.register_keras_serializable()
 class IndependentGamma(tfpl.DistributionLambda):
     """An independent gamma Keras layer."""
 
@@ -245,7 +245,7 @@ class IndependentGamma(tfpl.DistributionLambda):
         return super(IndependentGamma, self).output[0]
 
 
-@tf.keras.utils.register_keras_serializable()
+@tf.keras.saving.register_keras_serializable()
 class IndependentLogNormal(tfpl.DistributionLambda):
     """An independent LogNormal Keras layer."""
 
@@ -353,7 +353,7 @@ class IndependentLogNormal(tfpl.DistributionLambda):
         return super(IndependentLogNormal, self).output[0]
 
 
-@tf.keras.utils.register_keras_serializable()
+@tf.keras.saving.register_keras_serializable()
 class IndependentTruncatedNormal(tfpl.DistributionLambda):
     """An independent TruncatedNormal Keras layer."""
 
@@ -463,7 +463,7 @@ class IndependentTruncatedNormal(tfpl.DistributionLambda):
         return super(IndependentTruncatedNormal, self).output[0]
 
 
-@tf.keras.utils.register_keras_serializable()
+@tf.keras.saving.register_keras_serializable()
 class IndependentWeibull(tfpl.DistributionLambda):
     """An independent Weibull Keras layer."""
 
@@ -574,7 +574,7 @@ class IndependentWeibull(tfpl.DistributionLambda):
         return super(IndependentWeibull, self).output[0]
 
 
-@tf.keras.utils.register_keras_serializable()
+@tf.keras.saving.register_keras_serializable()
 class MultivariateNormalDiag(tfpl.DistributionLambda):
     """A `d`-variate normal Keras layer from `2* d` params,
     with a diagonal scale matrix.
@@ -670,7 +670,7 @@ class MultivariateNormalDiag(tfpl.DistributionLambda):
         return super(MultivariateNormalDiag, self).output[0]
 
 
-@tf.keras.utils.register_keras_serializable()
+@tf.keras.saving.register_keras_serializable()
 class MultivariateNormalTriL(tfpl.MultivariateNormalTriL):
     def __init__(
         self,

{mlpp_lib-0.5.2 → mlpp_lib-0.7.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "mlpp-lib"
-version = "0.5.2"
+version = "0.7.0"
 description = "Collection of methods for ML-based postprocessing of weather forecasts."
 authors = ["Daniele Nerini <daniele.nerini@meteoswiss.ch>"]
 include = ["LICENSE", "README.md"]
@@ -8,9 +8,9 @@ include = ["LICENSE", "README.md"]
 [tool.poetry.dependencies]
 python = ">=3.9, <3.11"
 numpy = "^1.22.1"
-tensorflow = {version = "~2.9.3", optional = true}
-tensorflow-cpu = {version = "~2.9.3", optional = true}
-tensorflow-probability = "~0.17.0"
+tensorflow = {version = "~2.12.1", optional = true}
+tensorflow-cpu = {version = "~2.12.1", optional = true}
+tensorflow-probability = "~0.20.0"
 pandas = "^1.4.1"
 scipy = "^1.8.0"
 xarray = "^2022.3.0"
@@ -25,10 +25,11 @@ tensorflow-cpu = ["tensorflow-cpu"]
 extended = ["zarr", "numba", "tensorflow"]
 
 [tool.poetry.group.dev.dependencies]
-tensorflow = "~2.9.3"
+tensorflow = "~2.12.1"
 pytest = "^7"
 black = "^22.1.0"
-mlflow = "<2"
+mlflow = "^2.8.1"
+jupyter = "^1.0.0"
 
 [build-system]
 requires = ["poetry-core>=1.0.8"]