ert 19.0.1__py3-none-any.whl → 20.0.0b0__py3-none-any.whl

ert/gui/tools/plot/plot_api.py

@@ -18,9 +18,12 @@ from pandas.api.types import is_numeric_dtype
 from pandas.errors import ParserError
 from resfo_utilities import history_key
 
-from ert.config import ParameterConfig, ResponseMetadata
-from ert.services import ErtServer
+from ert.config import ParameterConfig
+from ert.config.ensemble_config import ResponseConfig
+from ert.config.known_response_types import KnownResponseTypes
+from ert.services import create_ertserver_client
 from ert.storage.local_experiment import _parameters_adapter as parameter_config_adapter
+from ert.storage.local_experiment import _responses_adapter as response_config_adapter
 from ert.storage.realization_storage_state import RealizationStorageState
 
 logger = logging.getLogger(__name__)
@@ -46,18 +49,18 @@ class PlotApiKeyDefinition(NamedTuple):
     metadata: dict[Any, Any]
     filter_on: dict[Any, Any] | None = None
     parameter: ParameterConfig | None = None
-    response_metadata: ResponseMetadata | None = None
+    response: ResponseConfig | None = None
 
 
 class PlotApi:
     def __init__(self, ens_path: Path) -> None:
-        self.ens_path = ens_path
+        self.ens_path: Path = ens_path
         self._all_ensembles: list[EnsembleObject] | None = None
         self._timeout = 120
 
     @property
     def api_version(self) -> str:
-        with ErtServer.session(project=self.ens_path) as client:
+        with create_ertserver_client(self.ens_path) as client:
             try:
                 response = client.get("/version", timeout=self._timeout)
                 self._check_response(response)
@@ -83,7 +86,7 @@ class PlotApi:
             return self._all_ensembles
 
         self._all_ensembles = []
-        with ErtServer.session(project=self.ens_path) as client:
+        with create_ertserver_client(self.ens_path) as client:
             try:
                 response = client.get("/experiments", timeout=self._timeout)
                 self._check_response(response)
@@ -139,7 +142,7 @@ class PlotApi:
         all_keys: dict[str, PlotApiKeyDefinition] = {}
         all_params = {}
 
-        with ErtServer.session(project=self.ens_path) as client:
+        with create_ertserver_client(self.ens_path) as client:
             response = client.get("/experiments", timeout=self._timeout)
             self._check_response(response)
 
@@ -166,7 +169,7 @@ class PlotApi:
     def responses_api_key_defs(self) -> list[PlotApiKeyDefinition]:
         key_defs: dict[str, PlotApiKeyDefinition] = {}
 
-        with ErtServer.session(project=self.ens_path) as client:
+        with create_ertserver_client(self.ens_path) as client:
             response = client.get("/experiments", timeout=self._timeout)
             self._check_response(response)
 
@@ -176,22 +179,26 @@ class PlotApi:
                     key_defs[plot_key_def.key] = plot_key_def
 
             for experiment in response.json():
-                for response_type, response_metadatas in experiment[
-                    "responses"
-                ].items():
-                    for metadata in response_metadatas:
-                        key = metadata["response_key"]
+                for response_type, metadata in experiment["responses"].items():
+                    response_config: KnownResponseTypes = (
+                        response_config_adapter.validate_python(metadata)
+                    )
+                    keys = response_config.keys
+                    for key in keys:
                         has_obs = (
                             response_type in experiment["observations"]
                             and key in experiment["observations"][response_type]
                         )
-                        if metadata["filter_on"]:
+                        if response_config.filter_on is not None:
                             # Only assume one filter_on, this code is to be
                             # considered a bit "temp".
                             # In general, we could create a dropdown per
                             # filter_on on the frontend side
-                            for filter_key, values in metadata["filter_on"].items():
+
+                            filter_for_key = response_config.filter_on.get(key, {})
+                            for filter_key, values in filter_for_key.items():
                                 for v in values:
+                                    filter_on = {filter_key: v}
                                     subkey = f"{key}@{v}"
                                     update_keydef(
                                         PlotApiKeyDefinition(
@@ -202,10 +209,8 @@ class PlotApi:
                                             metadata={
                                                 "data_origin": response_type,
                                             },
-                                            filter_on={filter_key: v},
-                                            response_metadata=ResponseMetadata(
-                                                **metadata
-                                            ),
+                                            filter_on=filter_on,
+                                            response=response_config,
                                         )
                                     )
                         else:
@@ -216,7 +221,7 @@ class PlotApi:
                                     observations=has_obs,
                                     dimensionality=2,
                                     metadata={"data_origin": response_type},
-                                    response_metadata=ResponseMetadata(**metadata),
+                                    response=response_config,
                                 )
                             )
 
@@ -228,7 +233,9 @@ class PlotApi:
         response_key: str,
         filter_on: dict[str, Any] | None = None,
     ) -> pd.DataFrame:
-        with ErtServer.session(project=self.ens_path) as client:
+        if "@" in response_key:
+            response_key = response_key.split("@", maxsplit=1)[0]
+        with create_ertserver_client(self.ens_path) as client:
             response = client.get(
                 f"/ensembles/{ensemble_id}/responses/{PlotApi.escape(response_key)}",
                 headers={"accept": "application/x-parquet"},
@@ -256,7 +263,7 @@ class PlotApi:
                 return df
 
     def data_for_parameter(self, ensemble_id: str, parameter_key: str) -> pd.DataFrame:
-        with ErtServer.session(project=self.ens_path) as client:
+        with create_ertserver_client(self.ens_path) as client:
             parameter = client.get(
                 f"/ensembles/{ensemble_id}/parameters/{PlotApi.escape(parameter_key)}",
                 headers={"accept": "application/x-parquet"},
@@ -294,11 +301,12 @@ class PlotApi:
             )
             if not key_def:
                 raise httpx.RequestError(f"Response key {key_def} not found")
-
-            assert key_def.response_metadata is not None
-            actual_response_key = key_def.response_metadata.response_key
+            assert key_def.response is not None
+            actual_response_key = key
+            if "@" in actual_response_key:
+                actual_response_key = key.split("@", maxsplit=1)[0]
             filter_on = key_def.filter_on
-            with ErtServer.session(project=self.ens_path) as client:
+            with create_ertserver_client(self.ens_path) as client:
                 response = client.get(
                     f"/ensembles/{ensemble.id}/responses/{PlotApi.escape(actual_response_key)}/observations",
                     timeout=self._timeout,
@@ -386,7 +394,7 @@ class PlotApi:
         if not ensemble:
             return np.array([])
 
-        with ErtServer.session(project=self.ens_path) as client:
+        with create_ertserver_client(self.ens_path) as client:
             response = client.get(
                 f"/ensembles/{ensemble.id}/parameters/{PlotApi.escape(key)}/std_dev",
                 params={"z": z},

ert/gui/tools/plot/plot_widget.py

@@ -153,6 +153,7 @@ class PlotWidget(QWidget):
         # only for histogram plot see _sync_log_checkbox
         self._log_checkbox.setVisible(False)
         self._log_checkbox.setToolTip("Toggle data domain to log scale and back")
+        self._log_checkbox.clicked.connect(self.logLogScaleButtonUsage)
 
         log_checkbox_row = QHBoxLayout()
         log_checkbox_row.addWidget(self._log_checkbox)
@@ -193,6 +194,10 @@ class PlotWidget(QWidget):
     def name(self) -> str:
         return self._name
 
+    def logLogScaleButtonUsage(self) -> None:
+        logger.info(f"Plotwidget utility used: 'Log scale button' in tab '{self.name}'")
+        self._log_checkbox.clicked.disconnect()  # Log only once
+
     def updatePlot(
         self,
         plot_context: "PlotContext",

ert/gui/tools/plot/plot_window.py

@@ -25,7 +25,7 @@ from PyQt6.QtWidgets import (
 from ert.config.field import Field
 from ert.dark_storage.common import get_storage_api_version
 from ert.gui.ertwidgets import CopyButton, showWaitCursorWhileWaiting
-from ert.services._base_service import ServerBootFail
+from ert.services import ServerBootFail
 from ert.utils import log_duration
 
 from .customize import PlotCustomizer
@@ -267,10 +267,10 @@ class PlotWindow(QMainWindow):
             ensemble_to_data_map: dict[EnsembleObject, pd.DataFrame] = {}
             for ensemble in selected_ensembles:
                 try:
-                    if key_def.response_metadata is not None:
+                    if key_def.response is not None:
                         ensemble_to_data_map[ensemble] = self._api.data_for_response(
                             ensemble_id=ensemble.id,
-                            response_key=key_def.response_metadata.response_key,
+                            response_key=key,
                             filter_on=key_def.filter_on,
                         )
                     elif (
@@ -348,10 +348,7 @@ class PlotWindow(QMainWindow):
                     handle_exception(e)
                     plot_context.history_data = None
 
-            if (
-                key_def.response_metadata is not None
-                and key_def.response_metadata.response_type == "rft"
-            ):
+            if key_def.response is not None and key_def.response.type == "rft":
                 plot_context.setXLabel(key.split(":")[-1])
                 plot_context.setYLabel("TVD")
                 plot_context.depth_y_axis = True

ert/plugins/plugin_manager.py

@@ -338,6 +338,7 @@ class ErtRuntimePlugins(BaseModel):
     environment_variables: Mapping[str, str] = Field(default_factory=dict)
     env_pr_fm_step: Mapping[str, Mapping[str, Any]] = Field(default_factory=dict)
     help_links: dict[str, str] = Field(default_factory=dict)
+    prioritize_private_ip_address: bool = False
 
 
 def get_site_plugins(
@@ -386,6 +387,9 @@ def get_site_plugins(
         ),
         env_pr_fm_step=plugin_manager.get_forward_model_configuration(),
         help_links=plugin_manager.get_help_links(),
+        prioritize_private_ip_address=site_configurations.prioritize_private_ip_address
+        if site_configurations
+        else False,
     )
 
     return runtime_plugins

ert/run_models/ensemble_experiment.py

@@ -55,9 +55,7 @@ class EnsembleExperiment(InitialEnsembleRunModel, EnsembleExperimentConfig):
 
         experiment_storage = self._storage.create_experiment(
             parameters=cast(list[ParameterConfig], self.parameter_configuration),
-            observations={k: v.to_polars() for k, v in self.observations.items()}
-            if self.observations is not None
-            else None,
+            observations=self.observation_dataframes(),
             responses=cast(list[ResponseConfig], self.response_configuration),
             name=self.experiment_name,
             templates=self.ert_templates,

ert/run_models/ensemble_smoother.py

@@ -49,9 +49,7 @@ class EnsembleSmoother(InitialEnsembleRunModel, UpdateRunModel, EnsembleSmoother
 
         experiment_storage = self._storage.create_experiment(
             parameters=cast(list[ParameterConfig], self.parameter_configuration),
-            observations={k: v.to_polars() for k, v in self.observations.items()}
-            if self.observations is not None
-            else None,
+            observations=self.observation_dataframes(),
             responses=cast(list[ResponseConfig], self.response_configuration),
             name=self.experiment_name,
             templates=self.ert_templates,

ert/run_models/everest_run_model.py

@@ -831,19 +831,18 @@ class EverestRunModel(RunModel, EverestRunModelConfig):
 
         ensemble.save_everest_realization_info(realization_info)
 
-        for sim_id in range(sim_to_control_vector.shape[0]):
-            sim_controls = sim_to_control_vector[sim_id]
-            offset = 0
-            for control_config in self._everest_control_configs:
-                n_param_keys = len(control_config.parameter_keys)
-
-                # Save controls to ensemble
-                ensemble.save_parameters_numpy(
-                    sim_controls[offset : (offset + n_param_keys)].reshape(-1, 1),
-                    control_config.name,
-                    np.array([sim_id]),
-                )
-                offset += n_param_keys
+        iens = sim_to_control_vector.shape[0]
+        offset = 0
+        for control_config in self._everest_control_configs:
+            n_param_keys = len(control_config.parameter_keys)
+            name = control_config.name
+            parameters = sim_to_control_vector[:, offset : offset + n_param_keys]
+            ensemble.save_parameters_numpy(
+                parameters.reshape(-1, n_param_keys),
+                name,
+                np.arange(iens),
+            )
+            offset += n_param_keys
 
         # Evaluate the batch:
         run_args = self._get_run_args(

ert/run_models/initial_ensemble_run_model.py

@@ -3,15 +3,17 @@ from typing import Annotated, Any, Literal, Self
 import numpy as np
 import polars as pl
 from polars.datatypes import DataTypeClass
-from pydantic import BaseModel, Field, field_validator
+from pydantic import BaseModel, Field
 
 from ert.config import (
     EverestControl,
     GenKwConfig,
     KnownResponseTypes,
+    Observation,
     SurfaceConfig,
 )
 from ert.config import Field as FieldConfig
+from ert.config._create_observation_dataframes import create_observation_dataframes
 from ert.ensemble_evaluator.config import EvaluatorServerConfig
 from ert.run_arg import create_run_arguments
 from ert.run_models.run_model import RunModel, RunModelConfig
@@ -67,27 +69,7 @@ class InitialEnsembleRunModelConfig(RunModelConfig):
         ]
     ]
     ert_templates: list[tuple[str, str]]
-    observations: dict[str, DictEncodedDataFrame] | None = None
-
-    @field_validator("observations", mode="before")
-    @classmethod
-    def make_dict_encoded_observations(
-        cls, v: dict[str, pl.DataFrame | DictEncodedDataFrame | dict[str, Any]] | None
-    ) -> dict[str, DictEncodedDataFrame] | None:
-        if v is None:
-            return None
-
-        encoded = {}
-        for k, df in v.items():
-            match df:
-                case DictEncodedDataFrame():
-                    encoded[k] = df
-                case pl.DataFrame():
-                    encoded[k] = DictEncodedDataFrame.from_polars(df)
-                case dict():
-                    encoded[k] = DictEncodedDataFrame.model_validate(df)
-
-        return encoded
+    observations: list[Observation] | None = None
 
 
 class InitialEnsembleRunModel(RunModel, InitialEnsembleRunModelConfig):
@@ -101,6 +83,7 @@ class InitialEnsembleRunModel(RunModel, InitialEnsembleRunModelConfig):
             np.where(self.active_realizations)[0],
             parameters=[param.name for param in self.parameter_configuration],
             random_seed=self.random_seed,
+            num_realizations=self.runpath_config.num_realizations,
             design_matrix_df=self.design_matrix.to_polars()
             if self.design_matrix is not None
             else None,
@@ -117,3 +100,17 @@ class InitialEnsembleRunModel(RunModel, InitialEnsembleRunModelConfig):
             evaluator_server_config,
         )
         return ensemble_storage
+
+    def observation_dataframes(self) -> dict[str, pl.DataFrame]:
+        if self.observations is None:
+            return {}
+
+        rft_config = next(
+            (r for r in self.response_configuration if r.type == "rft"),
+            None,
+        )
+
+        return create_observation_dataframes(
+            observations=self.observations,
+            rft_config=rft_config,
+        )

ert/run_models/model_factory.py

@@ -153,7 +153,7 @@ def _setup_single_test_run(
         log_path=config.analysis_config.log_path,
         storage_path=config.ens_path,
         queue_config=config.queue_config.create_local_copy(),
-        observations=config.observations,
+        observations=config.observation_declarations,
     )
 
     return SingleTestRun(
@@ -212,7 +212,7 @@ def _setup_ensemble_experiment(
         log_path=config.analysis_config.log_path,
         storage_path=config.ens_path,
         queue_config=config.queue_config,
-        observations=config.observations,
+        observations=config.observation_declarations,
     )
 
     return EnsembleExperiment(
@@ -305,7 +305,7 @@ def _setup_manual_update(
         hooked_workflows=config.hooked_workflows,
         log_path=config.analysis_config.log_path,
         ert_templates=config.ert_templates,
-        observations=config.observations,
+        observations=config.observation_declarations,
     )
     return ManualUpdate(**runmodel_config.model_dump(), status_queue=status_queue)
 
@@ -343,7 +343,7 @@ def _setup_manual_update_enif(
         substitutions=config.substitutions,
         hooked_workflows=config.hooked_workflows,
         log_path=config.analysis_config.log_path,
-        observations=config.observations,
+        observations=config.observation_declarations,
     )
 
 
@@ -389,7 +389,7 @@ def _setup_ensemble_smoother(
         substitutions=config.substitutions,
         hooked_workflows=config.hooked_workflows,
         log_path=config.analysis_config.log_path,
-        observations=config.observations,
+        observations=config.observation_declarations,
     )
     return EnsembleSmoother(**runmodel_config.model_dump(), status_queue=status_queue)
 
@@ -435,7 +435,7 @@ def _setup_ensemble_information_filter(
         substitutions=config.substitutions,
         hooked_workflows=config.hooked_workflows,
         log_path=config.analysis_config.log_path,
-        observations=config.observations,
+        observations=config.observation_declarations,
     )
     return EnsembleInformationFilter(
         **runmodel_config.model_dump(), status_queue=status_queue
@@ -507,7 +507,7 @@ def _setup_multiple_data_assimilation(
         substitutions=config.substitutions,
         hooked_workflows=config.hooked_workflows,
         log_path=config.analysis_config.log_path,
-        observations=config.observations,
+        observations=config.observation_declarations,
     )
     return MultipleDataAssimilation(
         **runmodel_config.model_dump(), status_queue=status_queue

ert/run_models/multiple_data_assimilation.py

@@ -116,9 +116,7 @@ class MultipleDataAssimilation(
             sim_args = {"weights": self.weights}
             experiment_storage = self._storage.create_experiment(
                 parameters=cast(list[ParameterConfig], self.parameter_configuration),
-                observations={k: v.to_polars() for k, v in self.observations.items()}
-                if self.observations is not None
-                else None,
+                observations=self.observation_dataframes(),
                 responses=cast(list[ResponseConfig], self.response_configuration),
                 name=self.experiment_name,
                 templates=self.ert_templates,

ert/sample_prior.py

@@ -21,6 +21,7 @@ def sample_prior(
     ensemble: Ensemble,
     active_realizations: Iterable[int],
     random_seed: int,
+    num_realizations: int,
     parameters: list[str] | None = None,
     design_matrix_df: pl.DataFrame | None = None,
 ) -> None:
@@ -65,21 +66,18 @@ def sample_prior(
                     f"Sampling parameter {config_node.name} "
                     f"for realizations {active_realizations}"
                 )
-                datasets = [
-                    Ensemble.sample_parameter(
-                        config_node,
-                        realization_nr,
-                        random_seed=random_seed,
-                    )
-                    for realization_nr in active_realizations
-                ]
-                if datasets:
-                    dataset = pl.concat(datasets, how="vertical")
+                dataset = Ensemble.sample_parameter(
+                    config_node,
+                    list(active_realizations),
+                    random_seed=random_seed,
+                    num_realizations=num_realizations,
+                )
+            if not (dataset is None or dataset.is_empty()):
+                if complete_dataset is None:
+                    complete_dataset = dataset
+                elif dataset is not None:
+                    complete_dataset = complete_dataset.join(dataset, on="realization")
 
-            if complete_dataset is None:
-                complete_dataset = dataset
-            elif dataset is not None:
-                complete_dataset = complete_dataset.join(dataset, on="realization")
         else:
             for realization_nr in active_realizations:
                 ds = config_node.read_from_runpath(Path(), realization_nr, 0)

ert/services/__init__.py

@@ -1,4 +1,8 @@
-from .ert_server import ErtServer
-from .webviz_ert_service import WebvizErt
+from .ert_server import (
+    ErtServer,
+    ErtServerExit,
+    ServerBootFail,
+    create_ertserver_client,
+)
 
-__all__ = ["ErtServer", "WebvizErt"]
+__all__ = ["ErtServer", "ErtServerExit", "ServerBootFail", "create_ertserver_client"]