openstef 4.2.2__tar.gz → 4.3.0__tar.gz

{openstef-4.2.2 → openstef-4.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.4
+Metadata-Version: 2.5
 Name: openstef
-Version: 4.2.2
+Version: 4.3.0
 Summary: Open Short Term Energy forecasting library
 Project-URL: Documentation, https://openstef.github.io/openstef/index.html
 Project-URL: Homepage, https://lfenergy.org/projects/openstef/
@@ -16,26 +16,26 @@ Classifier: Programming Language :: Python :: 3 :: Only
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
 Requires-Python: <4.0,>=3.12
-Requires-Dist: openstef-core==4.2.2
-Requires-Dist: openstef-models[xgb-cpu]==4.2.2
+Requires-Dist: openstef-core==4.3.0
+Requires-Dist: openstef-models[xgb-cpu]==4.3.0
 Provides-Extra: all
-Requires-Dist: openstef-beam[all]==4.2.2; extra == 'all'
-Requires-Dist: openstef-core==4.2.2; extra == 'all'
-Requires-Dist: openstef-foundation-models[cpu]==4.2.2; extra == 'all'
-Requires-Dist: openstef-meta==4.2.2; extra == 'all'
-Requires-Dist: openstef-models[xgb-cpu]==4.2.2; extra == 'all'
+Requires-Dist: openstef-beam[all]==4.3.0; extra == 'all'
+Requires-Dist: openstef-core==4.3.0; extra == 'all'
+Requires-Dist: openstef-foundation-models[cpu]==4.3.0; extra == 'all'
+Requires-Dist: openstef-meta==4.3.0; extra == 'all'
+Requires-Dist: openstef-models[xgb-cpu]==4.3.0; extra == 'all'
 Provides-Extra: beam
 Requires-Dist: huggingface-hub>=0.35.3; extra == 'beam'
-Requires-Dist: openstef-beam==4.2.2; extra == 'beam'
+Requires-Dist: openstef-beam==4.3.0; extra == 'beam'
 Provides-Extra: core
-Requires-Dist: openstef-core==4.2.2; extra == 'core'
+Requires-Dist: openstef-core==4.3.0; extra == 'core'
 Provides-Extra: foundation-models
-Requires-Dist: openstef-foundation-models[cpu]==4.2.2; extra == 'foundation-models'
+Requires-Dist: openstef-foundation-models[cpu]==4.3.0; extra == 'foundation-models'
 Provides-Extra: meta
-Requires-Dist: openstef-meta==4.2.2; extra == 'meta'
+Requires-Dist: openstef-meta==4.3.0; extra == 'meta'
 Provides-Extra: models
-Requires-Dist: openstef-meta==4.2.2; extra == 'models'
-Requires-Dist: openstef-models[xgb-cpu]==4.2.2; extra == 'models'
+Requires-Dist: openstef-meta==4.3.0; extra == 'models'
+Requires-Dist: openstef-models[xgb-cpu]==4.3.0; extra == 'models'
 Description-Content-Type: text/markdown
 
 <!--

{openstef-4.2.2 → openstef-4.3.0}/docs/source/user_guide/guides/probabilistic_forecasting.rst

@@ -206,7 +206,44 @@ When ``use_local_quantile_estimation=True``, the calibrator estimates observed q
 
    The calibrator must be fitted (via ``workflow.fit()``) before it can be used for prediction. Calling ``predict()`` on a workflow with an unfitted calibrator will raise a :class:`~openstef_core.exceptions.NotFittedError`.
 
-For a complete worked example showing calibration before and after, including diagnostic plots, see :doc:`/tutorials/quantile_calibration`.
+For a complete worked example showing isotonic and asymmetric conformal
+calibration before and after, including diagnostic plots, see
+:doc:`/tutorials/quantile_calibration`.
+
+Conformalized Quantile Calibration
+----------------------------------
+
+OpenSTEF also provides a dependency-free
+:class:`~openstef_models.transforms.postprocessing.ConformalizedQuantileCalibrator`
+that implements asymmetric finite-sample conformal calibration. It is a
+postprocessing transform over a
+``ForecastDataset``:
+
+1. Fit the forecaster and generate forecasts for a time-ordered calibration
+   period.
+2. Fit the calibrator on those forecasts and the corresponding actuals.
+3. Apply the fitted corrections to later forecast datasets.
+
+The correction is asymmetric by tail. For a lower quantile ``q < 0.5``, the
+calibrator uses ``forecast - actual`` scores and subtracts the finite-sample
+quantile at ``1 - q``. For an upper quantile ``q >= 0.5``, it uses ``actual -
+forecast`` scores and adds the finite-sample quantile at ``q``. The finite-sample
+level is adjusted with ``min(level * (n + 1) / n, 1)`` and estimated with
+NumPy's ``method="higher"`` convention.
+
+By default, P50 is unchanged and quantile columns are not sorted inside the
+calibrator. This preserves the specified semantics: median calibration is an
+explicit opt-in, while downstream
+:class:`~openstef_models.transforms.postprocessing.quantile_sorter.QuantileSorter`
+owns the quantile-ordering invariant.
+
+.. code-block:: python
+
+   from openstef_models.transforms.postprocessing import ConformalizedQuantileCalibrator
+
+   calibrator = ConformalizedQuantileCalibrator(quantiles=quantiles)
+   calibrator.fit(calibration_forecasts_with_actuals)
+   calibrated_forecasts = calibrator.transform(forecasts)
 
 Evaluating Probabilistic Forecasts
 -----------------------------------

{openstef-4.2.2 → openstef-4.3.0}/examples/tutorials/quantile_calibration.ipynb

@@ -56,24 +56,25 @@
    "source": [
     "# Quantile Calibration\n",
     "\n",
-    "Improve the reliability of probabilistic forecasts using isotonic quantile\n",
-    "calibration.  A well-calibrated P10 quantile should exceed actual values\n",
-    "roughly 10 % of the time — this tutorial shows how to measure and correct\n",
-    "deviations.\n",
+    "Improve the reliability of probabilistic forecasts using isotonic and\n",
+    "asymmetric conformal\n",
+    "quantile calibration. A well-calibrated P10 quantile should exceed actual\n",
+    "values roughly 10 % of the time — this tutorial shows how to measure and\n",
+    "correct deviations.\n",
     "\n",
     "**What you'll learn:**\n",
     "\n",
     "- Measure quantile calibration with observed coverage\n",
-    "- Add isotonic calibration as a postprocessing step\n",
+    "- Add isotonic and asymmetric conformal calibration as postprocessing steps\n",
     "- Compare before/after calibration on real data\n",
     "\n",
     "```{note}\n",
     "This tutorial uses a small data slice for fast execution.\n",
-    "See `examples/benchmarks/` for production-scale runs.\n",
     "```\n",
     "\n",
     "**Key API references:**\n",
     "[`IsotonicQuantileCalibrator`](https://openstef.github.io/openstef/api/generated/openstef_models.transforms.postprocessing.IsotonicQuantileCalibrator.html)\n",
+    "· [`ConformalizedQuantileCalibrator`](https://openstef.github.io/openstef/api/generated/openstef_models.transforms.postprocessing.ConformalizedQuantileCalibrator.html)\n",
     "· [`ForecastingWorkflowConfig`](https://openstef.github.io/openstef/api/generated/openstef_models.presets.ForecastingWorkflowConfig.html)"
    ]
   },
@@ -111,11 +112,12 @@
     "\n",
     "train_start = datetime.fromisoformat(\"2024-03-01T00:00:00Z\")\n",
     "train_end = train_start + timedelta(days=45)\n",
-    "forecast_end = train_end + timedelta(days=7)\n",
+    "cal_end = train_end + timedelta(days=4)\n",
+    "forecast_end = cal_end + timedelta(days=7)\n",
     "\n",
     "train_dataset = dataset.filter_by_range(start=train_start, end=train_end)\n",
     "predict_dataset = dataset.filter_by_range(\n",
-    "    start=train_end - timedelta(days=14),\n",
+    "    start=cal_end - timedelta(days=14),\n",
     "    end=forecast_end,\n",
     ")\n",
     "\n",
@@ -139,7 +141,7 @@
     "\n",
     "workflow_uncal = create_forecasting_workflow(config=config)\n",
     "workflow_uncal.fit(train_dataset)\n",
-    "forecast_uncal = workflow_uncal.predict(predict_dataset, forecast_start=train_end)\n",
+    "forecast_uncal = workflow_uncal.predict(predict_dataset, forecast_start=cal_end)\n",
     "\n",
     "print(f\"Forecast rows: {len(forecast_uncal.data)}\")"
    ]
@@ -201,15 +203,16 @@
    "id": "c0b8a434",
    "metadata": {},
    "source": [
-    "## Add isotonic calibration\n",
-    "\n",
-    "[`IsotonicQuantileCalibrator`](https://openstef.github.io/openstef/api/generated/openstef_models.transforms.postprocessing.IsotonicQuantileCalibrator.html) is a postprocessing transform that learns a\n",
-    "monotonic mapping from predicted quantiles to observed quantile levels.\n",
-    "During training it fits on the validation split; during prediction it\n",
-    "corrects each quantile value.\n",
-    "\n",
-    "We create a second workflow identical to the first, but with the calibrator\n",
-    "appended to its postprocessing pipeline."
+    "## Add isotonic and asymmetric conformal calibration\n",
+    "\n",
+    "[`IsotonicQuantileCalibrator`](https://openstef.github.io/openstef/api/generated/openstef_models.transforms.postprocessing.IsotonicQuantileCalibrator.html) learns a\n",
+    "monotonic mapping from predicted quantiles to observed quantile levels. It is\n",
+    "appended to the workflow's postprocessing pipeline and fitted on training-set\n",
+    "predictions automatically.\n",
+    "[`ConformalizedQuantileCalibrator`](https://openstef.github.io/openstef/api/generated/openstef_models.transforms.postprocessing.ConformalizedQuantileCalibrator.html)\n",
+    "instead applies asymmetric conformal corrections. It leaves\n",
+    "P50 unchanged by default and delegates quantile ordering to a downstream\n",
+    "`QuantileSorter`."
    ]
   },
   {
@@ -233,7 +236,7 @@
     ")\n",
     "\n",
     "workflow_cal.fit(train_dataset)\n",
-    "forecast_cal = workflow_cal.predict(predict_dataset, forecast_start=train_end)"
+    "forecast_cal = workflow_cal.predict(predict_dataset, forecast_start=cal_end)"
    ]
   },
   {
@@ -250,12 +253,76 @@
     "assert len(forecast_cal.data) > 100, f\"Expected >100 calibrated forecast rows, got {len(forecast_cal.data)}\""
    ]
   },
+  {
+   "cell_type": "markdown",
+   "id": "4381f623",
+   "metadata": {},
+   "source": [
+    "### Calibrate with the asymmetric conformal transform\n",
+    "\n",
+    "Split-conformal calibration requires a **held-out** calibration period that\n",
+    "the forecaster was not trained on. We predict that period with the fitted\n",
+    "workflow, attach the observed target, and fit the calibrator before applying\n",
+    "it to the final holdout forecast."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "d6730356",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from openstef_models.transforms.postprocessing import ConformalizedQuantileCalibrator\n",
+    "\n",
+    "calibration_dataset = dataset.filter_by_range(\n",
+    "    start=train_end - timedelta(days=14),\n",
+    "    end=cal_end,\n",
+    ")\n",
+    "workflow_conformalized = create_forecasting_workflow(\n",
+    "    config=config.model_copy(update={\"model_id\": \"conformalized_gblinear\"})\n",
+    ")\n",
+    "workflow_conformalized.fit(train_dataset)\n",
+    "cal_forecast = workflow_conformalized.predict(calibration_dataset, forecast_start=train_end)\n",
+    "conformalized_calibrator = ConformalizedQuantileCalibrator(quantiles=quantiles)\n",
+    "conformalized_calibrator.fit(cal_forecast)\n",
+    "forecast_conformalized_raw = workflow_conformalized.predict(predict_dataset, forecast_start=cal_end)\n",
+    "forecast_conformalized = conformalized_calibrator.transform(forecast_conformalized_raw)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "f0b399bf",
+   "metadata": {
+    "tags": [
+     "remove-cell"
+    ]
+   },
+   "outputs": [],
+   "source": [
+    "assert len(forecast_conformalized.data) > 100, \"Expected >100 conformalized forecast rows\""
+   ]
+  },
   {
    "cell_type": "markdown",
    "id": "0dc57ac6",
    "metadata": {},
    "source": [
-    "## Compare calibration before and after"
+    "## Compare calibration before and after\n",
+    "\n",
+    "The following table and plot compare the uncalibrated forecast with both\n",
+    "postprocessing approaches on the same holdout rows.\n",
+    "\n",
+    "### Isotonic calibration\n",
+    "\n",
+    "Isotonic calibration learns a monotonic mapping for each quantile. The\n",
+    "``observed (isotonic)`` and ``error (isotonic)`` columns show its effect.\n",
+    "\n",
+    "### Asymmetric conformal calibration\n",
+    "\n",
+    "The ``observed (conformalized)`` and ``error (conformalized)`` columns show\n",
+    "the effect of the asymmetric corrections."
    ]
   },
   {
@@ -266,17 +333,23 @@
    "outputs": [],
    "source": [
     "forecast_cal_aligned = forecast_cal.data.loc[actuals.index]\n",
+    "forecast_conformalized_aligned = forecast_conformalized.data.loc[actuals.index]\n",
     "\n",
     "observed_cal = [float((actuals <= forecast_cal_aligned[f\"quantile_P{int(float(q) * 100)}\"]).mean()) for q in quantiles]\n",
+    "observed_conformalized = [\n",
+    "    float((actuals <= forecast_conformalized_aligned[f\"quantile_P{int(float(q) * 100)}\"]).mean()) for q in quantiles\n",
+    "]\n",
     "\n",
     "comparison_df = pd.DataFrame(\n",
     "    {\n",
     "        \"quantile\": [f\"P{int(float(q) * 100)}\" for q in quantiles],\n",
     "        \"expected\": expected,\n",
     "        \"observed (before)\": observed_uncal,\n",
-    "        \"observed (after)\": observed_cal,\n",
+    "        \"observed (isotonic)\": observed_cal,\n",
+    "        \"observed (conformalized)\": observed_conformalized,\n",
     "        \"error (before)\": [o - e for o, e in zip(observed_uncal, expected, strict=True)],\n",
-    "        \"error (after)\": [o - e for o, e in zip(observed_cal, expected, strict=True)],\n",
+    "        \"error (isotonic)\": [o - e for o, e in zip(observed_cal, expected, strict=True)],\n",
+    "        \"error (conformalized)\": [o - e for o, e in zip(observed_conformalized, expected, strict=True)],\n",
     "    }\n",
     ")\n",
     "print(comparison_df.to_string(index=False))"
@@ -321,12 +394,23 @@
     "        x=expected,\n",
     "        y=observed_cal,\n",
     "        mode=\"markers+lines\",\n",
-    "        name=\"After calibration\",\n",
+    "        name=\"After isotonic calibration\",\n",
     "        marker={\"size\": 12, \"color\": \"blue\"},\n",
     "        line={\"color\": \"blue\", \"width\": 2},\n",
     "    )\n",
     ")\n",
     "\n",
+    "fig.add_trace(\n",
+    "    go.Scatter(\n",
+    "        x=expected,\n",
+    "        y=observed_conformalized,\n",
+    "        mode=\"markers+lines\",\n",
+    "        name=\"After conformalized calibration\",\n",
+    "        marker={\"size\": 12, \"color\": \"green\", \"symbol\": \"diamond\"},\n",
+    "        line={\"color\": \"green\", \"width\": 2, \"dash\": \"dash\"},\n",
+    "    )\n",
+    ")\n",
+    "\n",
     "fig.update_layout(\n",
     "    title=\"Quantile calibration: expected vs observed coverage\",\n",
     "    xaxis_title=\"Expected quantile level\",\n",
@@ -344,10 +428,12 @@
    "id": "fb2c92a8",
    "metadata": {},
    "source": [
-    "Points closer to the diagonal indicate better calibration.  The isotonic\n",
-    "correction pulls the observed coverage towards the expected level, improving\n",
-    "the reliability of uncertainty estimates.  To measure calibration stability\n",
-    "over longer time horizons, combine this with a {doc}`backtesting_quickstart`."
+    "Points closer to the diagonal indicate better calibration. Isotonic regression\n",
+    "learns a monotonic value mapping, while the conformalized calibrator applies\n",
+    "asymmetric corrections. Compare both methods on a separate\n",
+    "holdout period before selecting one for production. To measure\n",
+    "calibration stability over longer time horizons, combine this with a\n",
+    "{doc}`backtesting_quickstart`."
    ]
   },
   {

{openstef-4.2.2 → openstef-4.3.0}/examples/tutorials/quantile_calibration.py

@@ -43,24 +43,25 @@ logger = setup_notebook_logging(
 # %% [markdown]
 # # Quantile Calibration
 #
-# Improve the reliability of probabilistic forecasts using isotonic quantile
-# calibration.  A well-calibrated P10 quantile should exceed actual values
-# roughly 10 % of the time — this tutorial shows how to measure and correct
-# deviations.
+# Improve the reliability of probabilistic forecasts using isotonic and
+# asymmetric conformal
+# quantile calibration. A well-calibrated P10 quantile should exceed actual
+# values roughly 10 % of the time — this tutorial shows how to measure and
+# correct deviations.
 #
 # **What you'll learn:**
 #
 # - Measure quantile calibration with observed coverage
-# - Add isotonic calibration as a postprocessing step
+# - Add isotonic and asymmetric conformal calibration as postprocessing steps
 # - Compare before/after calibration on real data
 #
 # ```{note}
 # This tutorial uses a small data slice for fast execution.
-# See `examples/benchmarks/` for production-scale runs.
 # ```
 #
 # **Key API references:**
 # [`IsotonicQuantileCalibrator`](https://openstef.github.io/openstef/api/generated/openstef_models.transforms.postprocessing.IsotonicQuantileCalibrator.html)
+# · [`ConformalizedQuantileCalibrator`](https://openstef.github.io/openstef/api/generated/openstef_models.transforms.postprocessing.ConformalizedQuantileCalibrator.html)
 # · [`ForecastingWorkflowConfig`](https://openstef.github.io/openstef/api/generated/openstef_models.presets.ForecastingWorkflowConfig.html)
 
 # %% [markdown]
@@ -86,11 +87,12 @@ dataset = load_liander_dataset()
 
 train_start = datetime.fromisoformat("2024-03-01T00:00:00Z")
 train_end = train_start + timedelta(days=45)
-forecast_end = train_end + timedelta(days=7)
+cal_end = train_end + timedelta(days=4)
+forecast_end = cal_end + timedelta(days=7)
 
 train_dataset = dataset.filter_by_range(start=train_start, end=train_end)
 predict_dataset = dataset.filter_by_range(
-    start=train_end - timedelta(days=14),
+    start=cal_end - timedelta(days=14),
     end=forecast_end,
 )
 
@@ -114,7 +116,7 @@ config = ForecastingWorkflowConfig(
 
 workflow_uncal = create_forecasting_workflow(config=config)
 workflow_uncal.fit(train_dataset)
-forecast_uncal = workflow_uncal.predict(predict_dataset, forecast_start=train_end)
+forecast_uncal = workflow_uncal.predict(predict_dataset, forecast_start=cal_end)
 
 print(f"Forecast rows: {len(forecast_uncal.data)}")
 
@@ -148,15 +150,16 @@ print("Calibration before isotonic correction:")
 print(calibration_df.to_string(index=False))
 
 # %% [markdown]
-# ## Add isotonic calibration
-#
-# [`IsotonicQuantileCalibrator`](https://openstef.github.io/openstef/api/generated/openstef_models.transforms.postprocessing.IsotonicQuantileCalibrator.html) is a postprocessing transform that learns a
-# monotonic mapping from predicted quantiles to observed quantile levels.
-# During training it fits on the validation split; during prediction it
-# corrects each quantile value.
+# ## Add isotonic and asymmetric conformal calibration
 #
-# We create a second workflow identical to the first, but with the calibrator
-# appended to its postprocessing pipeline.
+# [`IsotonicQuantileCalibrator`](https://openstef.github.io/openstef/api/generated/openstef_models.transforms.postprocessing.IsotonicQuantileCalibrator.html) learns a
+# monotonic mapping from predicted quantiles to observed quantile levels. It is
+# appended to the workflow's postprocessing pipeline and fitted on training-set
+# predictions automatically.
+# [`ConformalizedQuantileCalibrator`](https://openstef.github.io/openstef/api/generated/openstef_models.transforms.postprocessing.ConformalizedQuantileCalibrator.html)
+# instead applies asymmetric conformal corrections. It leaves
+# P50 unchanged by default and delegates quantile ordering to a downstream
+# `QuantileSorter`.
 
 # %%
 from openstef_models.transforms.postprocessing import IsotonicQuantileCalibrator
@@ -173,27 +176,74 @@ workflow_cal.model.postprocessing.transforms.append(
 )
 
 workflow_cal.fit(train_dataset)
-forecast_cal = workflow_cal.predict(predict_dataset, forecast_start=train_end)
+forecast_cal = workflow_cal.predict(predict_dataset, forecast_start=cal_end)
 
 # %% tags=["remove-cell"]
 assert len(forecast_cal.data) > 100, f"Expected >100 calibrated forecast rows, got {len(forecast_cal.data)}"
 
+# %% [markdown]
+# ### Calibrate with the asymmetric conformal transform
+#
+# Split-conformal calibration requires a **held-out** calibration period that
+# the forecaster was not trained on. We predict that period with the fitted
+# workflow, attach the observed target, and fit the calibrator before applying
+# it to the final holdout forecast.
+
+# %%
+from openstef_models.transforms.postprocessing import ConformalizedQuantileCalibrator
+
+calibration_dataset = dataset.filter_by_range(
+    start=train_end - timedelta(days=14),
+    end=cal_end,
+)
+workflow_conformalized = create_forecasting_workflow(
+    config=config.model_copy(update={"model_id": "conformalized_gblinear"})
+)
+workflow_conformalized.fit(train_dataset)
+cal_forecast = workflow_conformalized.predict(calibration_dataset, forecast_start=train_end)
+conformalized_calibrator = ConformalizedQuantileCalibrator(quantiles=quantiles)
+conformalized_calibrator.fit(cal_forecast)
+forecast_conformalized_raw = workflow_conformalized.predict(predict_dataset, forecast_start=cal_end)
+forecast_conformalized = conformalized_calibrator.transform(forecast_conformalized_raw)
+
+# %% tags=["remove-cell"]
+assert len(forecast_conformalized.data) > 100, "Expected >100 conformalized forecast rows"
+
 # %% [markdown]
 # ## Compare calibration before and after
+#
+# The following table and plot compare the uncalibrated forecast with both
+# postprocessing approaches on the same holdout rows.
+#
+# ### Isotonic calibration
+#
+# Isotonic calibration learns a monotonic mapping for each quantile. The
+# ``observed (isotonic)`` and ``error (isotonic)`` columns show its effect.
+#
+# ### Asymmetric conformal calibration
+#
+# The ``observed (conformalized)`` and ``error (conformalized)`` columns show
+# the effect of the asymmetric corrections.
 
 # %%
 forecast_cal_aligned = forecast_cal.data.loc[actuals.index]
+forecast_conformalized_aligned = forecast_conformalized.data.loc[actuals.index]
 
 observed_cal = [float((actuals <= forecast_cal_aligned[f"quantile_P{int(float(q) * 100)}"]).mean()) for q in quantiles]
+observed_conformalized = [
+    float((actuals <= forecast_conformalized_aligned[f"quantile_P{int(float(q) * 100)}"]).mean()) for q in quantiles
+]
 
 comparison_df = pd.DataFrame(
     {
         "quantile": [f"P{int(float(q) * 100)}" for q in quantiles],
         "expected": expected,
         "observed (before)": observed_uncal,
-        "observed (after)": observed_cal,
+        "observed (isotonic)": observed_cal,
+        "observed (conformalized)": observed_conformalized,
         "error (before)": [o - e for o, e in zip(observed_uncal, expected, strict=True)],
-        "error (after)": [o - e for o, e in zip(observed_cal, expected, strict=True)],
+        "error (isotonic)": [o - e for o, e in zip(observed_cal, expected, strict=True)],
+        "error (conformalized)": [o - e for o, e in zip(observed_conformalized, expected, strict=True)],
     }
 )
 print(comparison_df.to_string(index=False))
@@ -227,12 +277,23 @@ fig.add_trace(
         x=expected,
         y=observed_cal,
         mode="markers+lines",
-        name="After calibration",
+        name="After isotonic calibration",
         marker={"size": 12, "color": "blue"},
         line={"color": "blue", "width": 2},
     )
 )
 
+fig.add_trace(
+    go.Scatter(
+        x=expected,
+        y=observed_conformalized,
+        mode="markers+lines",
+        name="After conformalized calibration",
+        marker={"size": 12, "color": "green", "symbol": "diamond"},
+        line={"color": "green", "width": 2, "dash": "dash"},
+    )
+)
+
 fig.update_layout(
     title="Quantile calibration: expected vs observed coverage",
     xaxis_title="Expected quantile level",
@@ -245,10 +306,12 @@ fig.update_layout(
 fig.show()
 
 # %% [markdown]
-# Points closer to the diagonal indicate better calibration.  The isotonic
-# correction pulls the observed coverage towards the expected level, improving
-# the reliability of uncertainty estimates.  To measure calibration stability
-# over longer time horizons, combine this with a {doc}`backtesting_quickstart`.
+# Points closer to the diagonal indicate better calibration. Isotonic regression
+# learns a monotonic value mapping, while the conformalized calibrator applies
+# asymmetric corrections. Compare both methods on a separate
+# holdout period before selecting one for production. To measure
+# calibration stability over longer time horizons, combine this with a
+# {doc}`backtesting_quickstart`.
 
 # %% [markdown]
 # ## Next steps

{openstef-4.2.2 → openstef-4.3.0}/packages/openstef-beam/pyproject.toml

@@ -7,7 +7,7 @@ requires = [ "hatchling" ]
 
 [project]
 name = "openstef-beam"
-version = "4.2.2"
+version = "4.3.0"
 description = "Backtesting, Evaluation, Analysis and Metrics (BEAM) library for OpenSTEF"
 readme = "README.md"
 keywords = [ "energy", "forecasting", "machinelearning" ]

{openstef-4.2.2 → openstef-4.3.0}/packages/openstef-beam/src/openstef_beam/metrics/metrics_deterministic.py

@@ -19,7 +19,6 @@ from typing import NamedTuple
 
 import numpy as np
 import numpy.typing as npt
-from sklearn.metrics import r2_score
 
 from openstef_core.types import Quantile
 
@@ -518,16 +517,30 @@ def r2(
     well observed outcomes are replicated by the model, based on the proportion
     of total variation of outcomes explained by the model.
 
+    Structurally invalid inputs, such as incompatible shapes or unsupported
+    dimensions, raise ValueError. Statistically unusable data, such as fewer
+    than two samples or non-finite values, returns NaN.
+
     Args:
         y_true: Ground truth values with shape (num_samples,).
         y_pred: Predicted values with shape (num_samples,).
-        sample_weights: Optional weights for each sample with shape (num_samples,).
-            If None, all samples are weighted equally.
+        sample_weights: Optional weights for each sample with shape
+            (num_samples,). If None, all samples are weighted equally.
 
     Returns:
-        The R² score as a float. Best possible score is 1.0, and it can be negative
-        (because the model can be arbitrarily worse). A constant model that always
-        predicts the mean of y_true would get an R² score of 0.0.
+        The R² score as a float. The best possible score is 1.0, and the score
+        can be negative because a model can perform arbitrarily worse than a
+        constant mean predictor. Fewer than two samples or non-finite data
+        returns NaN.
+
+        For a constant target, this function follows scikit-learn's default
+        finite behavior: a perfect prediction returns 1.0 and an imperfect
+        prediction returns 0.0.
+
+    Raises:
+        ValueError: If the inputs are not one-dimensional, if y_true and y_pred
+            have different shapes, or if sample_weights does not have the same
+            one-dimensional shape as y_true.
 
     Example:
         Basic usage with energy load data
@@ -552,10 +565,58 @@ def r2(
         >>> isinstance(score, float)
         True
     """
-    if len(y_true) == 0 or len(y_pred) == 0:
-        return float("NaN")
+    y_true = np.asarray(y_true, dtype=float)
+    y_pred = np.asarray(y_pred, dtype=float)
+
+    # Structural contract violations should fail loudly. In particular,
+    # rejecting dimensions explicitly prevents unintended NumPy broadcasting.
+    if y_true.ndim != 1 or y_pred.ndim != 1:
+        error_msg = f"y_true and y_pred must be one-dimensional; got shapes {y_true.shape} and {y_pred.shape}"
+        raise ValueError(error_msg)
 
-    return float(r2_score(y_true, y_pred, sample_weight=sample_weights))
+    if y_true.shape != y_pred.shape:
+        error_msg = f"y_true and y_pred must have the same shape; got {y_true.shape} and {y_pred.shape}"
+        raise ValueError(error_msg)
+
+    if sample_weights is None:
+        weights = np.ones_like(y_true)
+    else:
+        weights = np.asarray(sample_weights, dtype=float)
+
+        if weights.ndim != 1 or weights.shape != y_true.shape:
+            error_msg = (
+                f"sample_weights must be one-dimensional and have the same "
+                f"shape as y_true; got {weights.shape} and {y_true.shape}"
+            )
+            raise ValueError(error_msg)
+
+    # R² is statistically undefined with fewer than two observations.
+    if y_true.size < 2:  # noqa: PLR2004 (hardcoded constant is too trivial to warrant a named constant)
+        return float("nan")
+
+    # Non-finite values represent unusable metric data rather than a structural
+    # programming error. Propagate that condition as an undefined metric.
+    if not (np.all(np.isfinite(y_true)) and np.all(np.isfinite(y_pred)) and np.all(np.isfinite(weights))):
+        return float("nan")
+
+    weight_sum = float(np.sum(weights))
+    if not np.isfinite(weight_sum) or weight_sum == 0.0:
+        return float("nan")
+
+    weighted_mean = float(np.average(y_true, weights=weights))
+    residual_sum = float(np.sum(weights * (y_true - y_pred) ** 2))
+    total_sum = float(np.sum(weights * (y_true - weighted_mean) ** 2))
+
+    if not np.isfinite(residual_sum) or not np.isfinite(total_sum):
+        return float("nan")
+
+    # Match scikit-learn's default force_finite=True behavior for a constant
+    # target: perfect predictions score 1.0, otherwise the score is 0.0.
+    if total_sum == 0.0:
+        return 1.0 if residual_sum == 0.0 else 0.0
+
+    score = 1.0 - residual_sum / total_sum
+    return float(score) if np.isfinite(score) else float("nan")
 
 
 def pinball_loss(