ert 18.0.9__py3-none-any.whl → 19.0.0rc0__py3-none-any.whl

ert/gui/tools/manage_experiments/storage_info_widget.py

@@ -1,3 +1,4 @@
+import contextlib
 import json
 from enum import IntEnum
 
@@ -6,13 +7,17 @@ import seaborn as sns
 import yaml
 from matplotlib.backends.backend_qt5agg import FigureCanvas  # type: ignore
 from matplotlib.figure import Figure
+from polars import DataFrame
 from PyQt6.QtCore import Qt
 from PyQt6.QtCore import pyqtSlot as Slot
 from PyQt6.QtWidgets import (
     QFrame,
     QHBoxLayout,
     QLabel,
+    QPushButton,
     QStackedLayout,
+    QTableWidget,
+    QTableWidgetItem,
     QTabWidget,
     QTextEdit,
     QTreeWidget,
@@ -21,8 +26,11 @@ from PyQt6.QtWidgets import (
     QWidget,
 )
 
+from ert import LibresFacade
 from ert.storage import Ensemble, Experiment, RealizationStorageState
 
+from .export_dialog import ExportDialog
+
 
 class _WidgetType(IntEnum):
     EMPTY_WIDGET = 0
@@ -42,6 +50,8 @@ class _EnsembleWidgetTabs(IntEnum):
     ENSEMBLE_TAB = 0
     STATE_TAB = 1
     OBSERVATIONS_TAB = 2
+    PARAMETERS_TAB = 3
+    MISFIT_TAB = 4
 
 
 class _ExperimentWidget(QWidget):
@@ -126,12 +136,15 @@ class _EnsembleWidget(QWidget):
 
         info_frame.setLayout(info_layout)
 
+        state_frame = QFrame()
+        state_layout = QHBoxLayout()
         self._state_text_edit = QTextEdit()
         self._state_text_edit.setReadOnly(True)
         self._state_text_edit.setObjectName("ensemble_state_text")
+        state_layout.addWidget(self._state_text_edit)
+        state_frame.setLayout(state_layout)
 
         observations_frame = QFrame()
-
         self._observations_tree_widget = QTreeWidget(self)
         self._observations_tree_widget.currentItemChanged.connect(
             self._currentItemChanged
@@ -151,15 +164,35 @@ class _EnsembleWidget(QWidget):
         observations_layout.addWidget(self._canvas)
         observations_frame.setLayout(observations_layout)
 
+        self._parameters_table = QTableWidget()
+        self._export_params_button = QPushButton("Export...")
+        self._export_params_button.clicked.connect(self.onClickExportParameters)
+
+        parameters_frame = self.create_export_frame(
+            self._parameters_table, self._export_params_button
+        )
+
+        self._misfit_table = QTableWidget()
+        self._export_misfit_button = QPushButton("Export...")
+        self._export_misfit_button.clicked.connect(self.onClickExportMisfit)
+
+        misfit_frame = self.create_export_frame(
+            self._misfit_table, self._export_misfit_button
+        )
+
         self._tab_widget = QTabWidget()
         self._tab_widget.insertTab(
             _EnsembleWidgetTabs.ENSEMBLE_TAB, info_frame, "Ensemble"
         )
+        self._tab_widget.insertTab(_EnsembleWidgetTabs.STATE_TAB, state_frame, "State")
         self._tab_widget.insertTab(
-            _EnsembleWidgetTabs.STATE_TAB, self._state_text_edit, "State"
+            _EnsembleWidgetTabs.OBSERVATIONS_TAB, observations_frame, "Observations"
         )
         self._tab_widget.insertTab(
-            _EnsembleWidgetTabs.OBSERVATIONS_TAB, observations_frame, "Observations"
+            _EnsembleWidgetTabs.PARAMETERS_TAB, parameters_frame, "Parameters"
+        )
+        self._tab_widget.insertTab(
+            _EnsembleWidgetTabs.MISFIT_TAB, misfit_frame, "Misfit"
         )
         self._tab_widget.currentChanged.connect(self._currentTabChanged)
 
@@ -168,6 +201,17 @@ class _EnsembleWidget(QWidget):
 
         self.setLayout(layout)
 
+    def create_export_frame(self, table: QTableWidget, button: QPushButton) -> QFrame:
+        export_frame = QFrame()
+        export_layout = QVBoxLayout()
+        vertical_header = table.verticalHeader()
+        assert vertical_header is not None
+        vertical_header.setVisible(False)
+        export_layout.addWidget(table)
+        export_layout.addWidget(button)
+        export_frame.setLayout(export_layout)
+        return export_frame
+
     def _currentItemChanged(
         self, selected: QTreeWidgetItem, _: QTreeWidgetItem
     ) -> None:
@@ -356,12 +400,53 @@ class _EnsembleWidget(QWidget):
                         0, Qt.SortOrder.AscendingOrder
                     )
 
-        for i in range(self._observations_tree_widget.topLevelItemCount()):
-            item = self._observations_tree_widget.topLevelItem(i)
-            assert item is not None
-            if item.childCount() > 0:
-                self._observations_tree_widget.setCurrentItem(item.child(0))
-                break
+            for i in range(self._observations_tree_widget.topLevelItemCount()):
+                item = self._observations_tree_widget.topLevelItem(i)
+                assert item is not None
+                if item.childCount() > 0:
+                    self._observations_tree_widget.setCurrentItem(item.child(0))
+                    break
+
+        elif index in {
+            _EnsembleWidgetTabs.PARAMETERS_TAB,
+            _EnsembleWidgetTabs.MISFIT_TAB,
+        }:
+            assert self._ensemble is not None
+
+            df: pl.DataFrame = pl.DataFrame()
+            with contextlib.suppress(Exception):
+                if index == _EnsembleWidgetTabs.PARAMETERS_TAB:
+                    df = self._ensemble.load_scalar_keys(transformed=True)
+                else:
+                    df = self.get_misfit_df()
+
+            table = (
+                self._parameters_table
+                if index == _EnsembleWidgetTabs.PARAMETERS_TAB
+                else self._misfit_table
+            )
+
+            table.setUpdatesEnabled(False)
+            table.setSortingEnabled(False)
+            table.setRowCount(df.height)
+            table.setColumnCount(df.width)
+            table.setHorizontalHeaderLabels(df.columns)
+
+            rows = df.rows()
+            for r, row in enumerate(rows):
+                for c, v in enumerate(row):
+                    table.setItem(r, c, QTableWidgetItem("" if v is None else str(v)))
+
+            table.resizeColumnsToContents()
+            table.setUpdatesEnabled(True)
+
+    def get_misfit_df(self) -> DataFrame:
+        assert self._ensemble is not None
+        df = LibresFacade.load_all_misfit_data(self._ensemble)
+        realization_column = pl.Series(df.index)
+        df = pl.from_pandas(df)
+        df.insert_column(0, realization_column)
+        return df
 
     @Slot(Ensemble)
     def setEnsemble(self, ensemble: Ensemble) -> None:
@@ -372,6 +457,22 @@ class _EnsembleWidget(QWidget):
 
         self._tab_widget.setCurrentIndex(0)
 
+    @Slot()
+    def onClickExportMisfit(self) -> None:
+        assert self._ensemble is not None
+        misfit_df = self.get_misfit_df()
+        export_dialog = ExportDialog(misfit_df, "Export misfit", parent=self)
+        export_dialog.show()
+
+    @Slot()
+    def onClickExportParameters(self) -> None:
+        assert self._ensemble is not None
+        parameters_df = self._ensemble.load_scalar_keys(transformed=True)
+        export_dialog = ExportDialog(
+            parameters_df, window_title="Export parameters", parent=self
+        )
+        export_dialog.show()
+
 
 class _RealizationWidget(QWidget):
     def __init__(self) -> None:

ert/gui/tools/plot/plot_api.py

@@ -18,8 +18,9 @@ from pandas.api.types import is_numeric_dtype
 from pandas.errors import ParserError
 from resfo_utilities import history_key
 
-from ert.config import ParameterMetadata, ResponseMetadata
+from ert.config import ParameterConfig, ResponseMetadata
 from ert.services import ErtServer
+from ert.storage.local_experiment import _parameters_adapter as parameter_config_adapter
 from ert.storage.realization_storage_state import RealizationStorageState
 
 logger = logging.getLogger(__name__)
@@ -44,7 +45,7 @@ class PlotApiKeyDefinition(NamedTuple):
     dimensionality: int
     metadata: dict[Any, Any]
     filter_on: dict[Any, Any] | None = None
-    parameter_metadata: ParameterMetadata | None = None
+    parameter: ParameterConfig | None = None
     response_metadata: ResponseMetadata | None = None
 
 
@@ -143,18 +144,21 @@ class PlotApi:
             self._check_response(response)
 
             for experiment in response.json():
-                for param_metadatas in experiment["parameters"].values():
-                    for metadata in param_metadatas:
-                        param_key = metadata["key"]
-                        all_keys[param_key] = PlotApiKeyDefinition(
-                            key=param_key,
-                            index_type=None,
-                            observations=False,
-                            dimensionality=metadata["dimensionality"],
-                            metadata=metadata["userdata"],
-                            parameter_metadata=ParameterMetadata(**metadata),
-                        )
-                        all_params[param_key] = all_keys[param_key]
+                for metadata in experiment["parameters"].values():
+                    param_cfg = parameter_config_adapter.validate_python(metadata)
+                    if group := metadata.get("group"):
+                        param_key = f"{group}:{metadata['name']}"
+                    else:
+                        param_key = metadata["name"]
+                    all_keys[param_key] = PlotApiKeyDefinition(
+                        key=param_key,
+                        index_type=None,
+                        observations=False,
+                        dimensionality=metadata["dimensionality"],
+                        metadata={"data_origin": metadata["type"]},
+                        parameter=param_cfg,
+                    )
+                    all_params[param_key] = all_keys[param_key]
 
         return list(all_keys.values())
 
@@ -318,12 +322,17 @@ class PlotApi:
                         f"ensemble_name={ensemble.name}, e={e}"
                     ) from e
 
+                key_index: list[int | float | pd.Timestamp]
                 for obs in observations:
                     try:
                         int(obs["x_axis"][0])
                         key_index = [int(v) for v in obs["x_axis"]]
                     except ValueError:
-                        key_index = [pd.Timestamp(v) for v in obs["x_axis"]]
+                        try:
+                            float(obs["x_axis"][0])
+                            key_index = [float(v) for v in obs["x_axis"]]
+                        except ValueError:
+                            key_index = [pd.Timestamp(v) for v in obs["x_axis"]]
 
                     observations_dfs.append(
                         pd.DataFrame(

ert/gui/tools/plot/plot_widget.py

@@ -25,7 +25,7 @@ from PyQt6.QtWidgets import (
 )
 from typing_extensions import override
 
-from .plot_api import EnsembleObject
+from .plot_api import EnsembleObject, PlotApiKeyDefinition
 
 if TYPE_CHECKING:
     from .plottery import PlotContext
@@ -34,6 +34,7 @@ if TYPE_CHECKING:
     from .plottery.plots.ensemble import EnsemblePlot
     from .plottery.plots.gaussian_kde import GaussianKDEPlot
     from .plottery.plots.histogram import HistogramPlot
+    from .plottery.plots.misfits import MisfitsPlot
     from .plottery.plots.statistics import StatisticsPlot
     from .plottery.plots.std_dev import StdDevPlot
 
@@ -122,6 +123,7 @@ class PlotWidget(QWidget):
             "DistributionPlot",
             "CrossEnsembleStatisticsPlot",
             "StdDevPlot",
+            "MisfitsPlot",
         ],
         parent: QWidget | None = None,
     ) -> None:
@@ -173,7 +175,11 @@ class PlotWidget(QWidget):
         self._figure.clear()
 
     def _sync_log_checkbox(self) -> None:
-        if type(self._plotter).__name__ == "HistogramPlot":
+        if type(self._plotter).__name__ in {
+            "HistogramPlot",
+            "DistributionPlot",
+            "GaussianKDEPlot",
+        }:
             self._log_checkbox.setVisible(True)
         else:
             self._log_checkbox.setVisible(False)
@@ -188,6 +194,7 @@ class PlotWidget(QWidget):
         ensemble_to_data_map: dict[EnsembleObject, pd.DataFrame],
         observations: pd.DataFrame,
         std_dev_images: dict[str, npt.NDArray[np.float32]],
+        key_def: PlotApiKeyDefinition | None = None,
     ) -> None:
         self.resetPlot()
         try:
@@ -200,6 +207,7 @@ class PlotWidget(QWidget):
                 ensemble_to_data_map,
                 observations,
                 std_dev_images,
+                key_def,
             )
             self._canvas.draw()
             self._sync_log_checkbox()

ert/gui/tools/plot/plot_window.py

@@ -22,6 +22,7 @@ from PyQt6.QtWidgets import (
     QWidget,
 )
 
+from ert.config.field import Field
 from ert.dark_storage.common import get_storage_api_version
 from ert.gui.ertwidgets import CopyButton, showWaitCursorWhileWaiting
 from ert.services._base_service import ServerBootFail
@@ -39,6 +40,7 @@ from .plottery.plots import (
     EnsemblePlot,
     GaussianKDEPlot,
     HistogramPlot,
+    MisfitsPlot,
     StatisticsPlot,
     StdDevPlot,
 )
@@ -50,10 +52,11 @@ ENSEMBLE = "Ensemble"
 HISTOGRAM = "Histogram"
 STATISTICS = "Statistics"
 STD_DEV = "Std Dev"
+MISFITS = "Misfits"
 
 RESPONSE_DEFAULT = 0
-GEN_KW_DEFAULT = 2
-STD_DEV_DEFAULT = 6
+GEN_KW_DEFAULT = 3
+STD_DEV_DEFAULT = 7
 
 
 logger = logging.getLogger(__name__)
@@ -188,6 +191,7 @@ class PlotWindow(QMainWindow):
 
             self.addPlotWidget(ENSEMBLE, EnsemblePlot())
             self.addPlotWidget(STATISTICS, StatisticsPlot())
+            self.addPlotWidget(MISFITS, MisfitsPlot())
             self.addPlotWidget(HISTOGRAM, HistogramPlot())
             self.addPlotWidget(GAUSSIAN_KDE, GaussianKDEPlot())
             self.addPlotWidget(DISTRIBUTION, DistributionPlot())
@@ -270,12 +274,12 @@ class PlotWindow(QMainWindow):
                             filter_on=key_def.filter_on,
                         )
                     elif (
-                        key_def.parameter_metadata is not None
-                        and "GEN_KW" in key_def.metadata["data_origin"]
+                        key_def.parameter is not None
+                        and key_def.parameter.type == "gen_kw"
                     ):
                         ensemble_to_data_map[ensemble] = self._api.data_for_parameter(
                             ensemble_id=ensemble.id,
-                            parameter_key=key_def.parameter_metadata.key,
+                            parameter_key=key_def.parameter.name,
                         )
                 except BaseException as e:
                     handle_exception(e)
@@ -290,10 +294,10 @@ class PlotWindow(QMainWindow):
                     handle_exception(e)
 
             std_dev_images: dict[str, npt.NDArray[np.float32]] = {}
-            if "FIELD" in key_def.metadata["data_origin"]:
-                plot_widget.showLayerWidget.emit(True)
 
-                layers = key_def.metadata["ertbox_params"]["nz"]
+            if isinstance(key_def.parameter, Field):
+                plot_widget.showLayerWidget.emit(True)
+                layers = key_def.parameter.ertbox_params.nz
                 plot_widget.updateLayerWidget.emit(layers)
 
                 if layer is None:
@@ -357,7 +361,11 @@ class PlotWindow(QMainWindow):
             self._updateCustomizer(plot_widget, self._preferred_ensemble_x_axis_format)
 
             plot_widget.updatePlot(
-                plot_context, ensemble_to_data_map, observations, std_dev_images
+                plot_context,
+                ensemble_to_data_map,
+                observations,
+                std_dev_images,
+                key_def,
             )
 
     def _updateCustomizer(
@@ -386,15 +394,14 @@ class PlotWindow(QMainWindow):
     def addPlotWidget(
         self,
         name: str,
-        plotter: (
-            EnsemblePlot
-            | StatisticsPlot
-            | HistogramPlot
-            | GaussianKDEPlot
-            | DistributionPlot
-            | CrossEnsembleStatisticsPlot
-            | StdDevPlot
-        ),
+        plotter: EnsemblePlot
+        | StatisticsPlot
+        | HistogramPlot
+        | GaussianKDEPlot
+        | DistributionPlot
+        | CrossEnsembleStatisticsPlot
+        | StdDevPlot
+        | MisfitsPlot,
         enabled: bool = True,
     ) -> None:
         plot_widget = PlotWidget(name, plotter)
@@ -433,6 +440,7 @@ class PlotWindow(QMainWindow):
             widget
             for widget in self._plot_widgets
             if widget._plotter.dimensionality == key_def.dimensionality
+            and (key_def.observations or not widget._plotter.requires_observations)
         ]
 
         # Enabling/disabling tab triggers the

ert/gui/tools/plot/plottery/plots/__init__.py

@@ -3,6 +3,7 @@ from .distribution import DistributionPlot
 from .ensemble import EnsemblePlot
 from .gaussian_kde import GaussianKDEPlot
 from .histogram import HistogramPlot
+from .misfits import MisfitsPlot
 from .statistics import StatisticsPlot
 from .std_dev import StdDevPlot
 
@@ -12,6 +13,7 @@ __all__ = [
     "EnsemblePlot",
     "GaussianKDEPlot",
     "HistogramPlot",
+    "MisfitsPlot",
     "StatisticsPlot",
     "StdDevPlot",
 ]

ert/gui/tools/plot/plottery/plots/cesp.py

@@ -8,7 +8,7 @@ from matplotlib.lines import Line2D
 from matplotlib.patches import Rectangle
 from typing_extensions import TypedDict
 
-from ert.gui.tools.plot.plot_api import EnsembleObject
+from ert.gui.tools.plot.plot_api import EnsembleObject, PlotApiKeyDefinition
 
 from .plot_tools import ConditionalAxisFormatter, PlotTools
 
@@ -37,6 +37,7 @@ class CcsData(TypedDict):
 class CrossEnsembleStatisticsPlot:
     def __init__(self) -> None:
         self.dimensionality = 1
+        self.requires_observations = False
 
     @staticmethod
     def plot(
@@ -45,6 +46,7 @@ class CrossEnsembleStatisticsPlot:
         ensemble_to_data_map: dict[EnsembleObject, pd.DataFrame],
         observation_data: pd.DataFrame,
         std_dev_images: dict[str, npt.NDArray[np.float32]],
+        key_def: PlotApiKeyDefinition | None = None,
     ) -> None:
         plotCrossEnsembleStatistics(
             figure, plot_context, ensemble_to_data_map, observation_data

ert/gui/tools/plot/plottery/plots/distribution.py

@@ -5,7 +5,7 @@ from typing import TYPE_CHECKING
 import numpy as np
 import pandas as pd
 
-from ert.gui.tools.plot.plot_api import EnsembleObject
+from ert.gui.tools.plot.plot_api import EnsembleObject, PlotApiKeyDefinition
 
 from .plot_tools import ConditionalAxisFormatter, PlotTools
 
@@ -20,6 +20,7 @@ if TYPE_CHECKING:
 class DistributionPlot:
     def __init__(self) -> None:
         self.dimensionality = 1
+        self.requires_observations = False
 
     @staticmethod
     def plot(
@@ -28,6 +29,7 @@ class DistributionPlot:
         ensemble_to_data_map: dict[EnsembleObject, pd.DataFrame],
         observation_data: pd.DataFrame,
         std_dev_images: dict[str, npt.NDArray[np.float32]],
+        key_def: PlotApiKeyDefinition | None = None,
     ) -> None:
         plotDistribution(figure, plot_context, ensemble_to_data_map, observation_data)
 
@@ -80,6 +82,9 @@ def plotDistribution(
     )
     config.setLegendEnabled(False)
 
+    if plot_context.log_scale:
+        axes.set_yscale("log")
+
     PlotTools.finalizePlot(
         plot_context, figure, axes, default_x_label="Ensemble", default_y_label="Value"
     )

ert/gui/tools/plot/plottery/plots/ensemble.py

@@ -15,13 +15,14 @@ if TYPE_CHECKING:
     from matplotlib.axes import Axes
     from matplotlib.figure import Figure
 
-    from ert.gui.tools.plot.plot_api import EnsembleObject
+    from ert.gui.tools.plot.plot_api import EnsembleObject, PlotApiKeyDefinition
     from ert.gui.tools.plot.plottery import PlotConfig, PlotContext
 
 
 class EnsemblePlot:
     def __init__(self) -> None:
         self.dimensionality = 2
+        self.requires_observations = False
 
     def plot(
         self,
@@ -30,6 +31,7 @@ class EnsemblePlot:
         ensemble_to_data_map: dict[EnsembleObject, pd.DataFrame],
         observation_data: pd.DataFrame,
         std_dev_images: dict[str, npt.NDArray[np.float32]],
+        key_def: PlotApiKeyDefinition | None = None,
     ) -> None:
         config = plot_context.plotConfig()
         axes = figure.add_subplot(111)

ert/gui/tools/plot/plottery/plots/gaussian_kde.py

@@ -6,7 +6,7 @@ import numpy as np
 import pandas as pd
 from scipy.stats import gaussian_kde
 
-from ert.gui.tools.plot.plot_api import EnsembleObject
+from ert.gui.tools.plot.plot_api import EnsembleObject, PlotApiKeyDefinition
 
 from .plot_tools import ConditionalAxisFormatter, PlotTools
 
@@ -21,6 +21,7 @@ if TYPE_CHECKING:
 class GaussianKDEPlot:
     def __init__(self) -> None:
         self.dimensionality = 1
+        self.requires_observations = False
 
     @staticmethod
     def plot(
@@ -29,10 +30,16 @@ class GaussianKDEPlot:
         ensemble_to_data_map: dict[EnsembleObject, pd.DataFrame],
         observation_data: pd.DataFrame,
         std_dev_images: dict[str, npt.NDArray[np.float32]],
+        key_def: PlotApiKeyDefinition | None = None,
     ) -> None:
         plotGaussianKDE(figure, plot_context, ensemble_to_data_map, observation_data)
 
 
+def _array_is_constant(data: pd.Series | pd.DataFrame) -> bool:
+    array = data.to_numpy()
+    return array.shape[0] == 0 or (array[0] == array).all()
+
+
 def plotGaussianKDE(
     figure: Figure,
     plot_context: PlotContext,
@@ -55,11 +62,14 @@ def plotGaussianKDE(
         if data.empty:
             continue
         data = data[0]
-        if data.nunique() > 1:
+        if not _array_is_constant(data):
             _plotGaussianKDE(
                 axes, config, data, f"{ensemble.experiment_name} : {ensemble.name}"
             )
 
+    if plot_context.log_scale:
+        axes.set_xscale("log")
+
     PlotTools.finalizePlot(
         plot_context, figure, axes, default_x_label="Value", default_y_label="Density"
     )

ert/gui/tools/plot/plottery/plots/histogram.py

@@ -8,7 +8,7 @@ import numpy as np
 import pandas as pd
 from matplotlib.patches import Rectangle
 
-from ert.gui.tools.plot.plot_api import EnsembleObject
+from ert.gui.tools.plot.plot_api import EnsembleObject, PlotApiKeyDefinition
 from ert.shared.status.utils import convert_to_numeric
 
 from .plot_tools import ConditionalAxisFormatter, PlotTools
@@ -24,6 +24,7 @@ if TYPE_CHECKING:
 class HistogramPlot:
     def __init__(self) -> None:
         self.dimensionality = 1
+        self.requires_observations = False
 
     @staticmethod
     def plot(
@@ -32,6 +33,7 @@ class HistogramPlot:
         ensemble_to_data_map: dict[EnsembleObject, pd.DataFrame],
         observation_data: pd.DataFrame,
         std_dev_images: dict[str, npt.NDArray[np.float32]],
+        key_def: PlotApiKeyDefinition | None = None,
     ) -> None:
         plotHistogram(figure, plot_context, ensemble_to_data_map)