ert 19.0.1__py3-none-any.whl → 20.0.0b1__py3-none-any.whl

ert/gui/tools/plot/plot_api.py

@@ -18,9 +18,12 @@ from pandas.api.types import is_numeric_dtype
 from pandas.errors import ParserError
 from resfo_utilities import history_key
 
-from ert.config import ParameterConfig, ResponseMetadata
-from ert.services import ErtServer
+from ert.config import ParameterConfig
+from ert.config.ensemble_config import ResponseConfig
+from ert.config.known_response_types import KnownResponseTypes
+from ert.services import create_ertserver_client
 from ert.storage.local_experiment import _parameters_adapter as parameter_config_adapter
+from ert.storage.local_experiment import _responses_adapter as response_config_adapter
 from ert.storage.realization_storage_state import RealizationStorageState
 
 logger = logging.getLogger(__name__)
@@ -46,18 +49,18 @@ class PlotApiKeyDefinition(NamedTuple):
     metadata: dict[Any, Any]
     filter_on: dict[Any, Any] | None = None
     parameter: ParameterConfig | None = None
-    response_metadata: ResponseMetadata | None = None
+    response: ResponseConfig | None = None
 
 
 class PlotApi:
     def __init__(self, ens_path: Path) -> None:
-        self.ens_path = ens_path
+        self.ens_path: Path = ens_path
         self._all_ensembles: list[EnsembleObject] | None = None
         self._timeout = 120
 
     @property
     def api_version(self) -> str:
-        with ErtServer.session(project=self.ens_path) as client:
+        with create_ertserver_client(self.ens_path) as client:
             try:
                 response = client.get("/version", timeout=self._timeout)
                 self._check_response(response)
@@ -83,7 +86,7 @@ class PlotApi:
             return self._all_ensembles
 
         self._all_ensembles = []
-        with ErtServer.session(project=self.ens_path) as client:
+        with create_ertserver_client(self.ens_path) as client:
             try:
                 response = client.get("/experiments", timeout=self._timeout)
                 self._check_response(response)
@@ -139,7 +142,7 @@ class PlotApi:
         all_keys: dict[str, PlotApiKeyDefinition] = {}
         all_params = {}
 
-        with ErtServer.session(project=self.ens_path) as client:
+        with create_ertserver_client(self.ens_path) as client:
             response = client.get("/experiments", timeout=self._timeout)
             self._check_response(response)
 
@@ -166,7 +169,7 @@ class PlotApi:
     def responses_api_key_defs(self) -> list[PlotApiKeyDefinition]:
         key_defs: dict[str, PlotApiKeyDefinition] = {}
 
-        with ErtServer.session(project=self.ens_path) as client:
+        with create_ertserver_client(self.ens_path) as client:
             response = client.get("/experiments", timeout=self._timeout)
             self._check_response(response)
 
@@ -176,22 +179,26 @@ class PlotApi:
                     key_defs[plot_key_def.key] = plot_key_def
 
             for experiment in response.json():
-                for response_type, response_metadatas in experiment[
-                    "responses"
-                ].items():
-                    for metadata in response_metadatas:
-                        key = metadata["response_key"]
+                for response_type, metadata in experiment["responses"].items():
+                    response_config: KnownResponseTypes = (
+                        response_config_adapter.validate_python(metadata)
+                    )
+                    keys = response_config.keys
+                    for key in keys:
                         has_obs = (
                             response_type in experiment["observations"]
                             and key in experiment["observations"][response_type]
                         )
-                        if metadata["filter_on"]:
+                        if response_config.filter_on is not None:
                             # Only assume one filter_on, this code is to be
                             # considered a bit "temp".
                             # In general, we could create a dropdown per
                             # filter_on on the frontend side
-                            for filter_key, values in metadata["filter_on"].items():
+
+                            filter_for_key = response_config.filter_on.get(key, {})
+                            for filter_key, values in filter_for_key.items():
                                 for v in values:
+                                    filter_on = {filter_key: v}
                                     subkey = f"{key}@{v}"
                                     update_keydef(
                                         PlotApiKeyDefinition(
@@ -202,10 +209,8 @@ class PlotApi:
                                             metadata={
                                                 "data_origin": response_type,
                                             },
-                                            filter_on={filter_key: v},
-                                            response_metadata=ResponseMetadata(
-                                                **metadata
-                                            ),
+                                            filter_on=filter_on,
+                                            response=response_config,
                                         )
                                     )
                         else:
@@ -216,7 +221,7 @@ class PlotApi:
                                     observations=has_obs,
                                     dimensionality=2,
                                     metadata={"data_origin": response_type},
-                                    response_metadata=ResponseMetadata(**metadata),
+                                    response=response_config,
                                 )
                             )
 
@@ -228,7 +233,9 @@ class PlotApi:
         response_key: str,
         filter_on: dict[str, Any] | None = None,
     ) -> pd.DataFrame:
-        with ErtServer.session(project=self.ens_path) as client:
+        if "@" in response_key:
+            response_key = response_key.split("@", maxsplit=1)[0]
+        with create_ertserver_client(self.ens_path) as client:
             response = client.get(
                 f"/ensembles/{ensemble_id}/responses/{PlotApi.escape(response_key)}",
                 headers={"accept": "application/x-parquet"},
@@ -256,7 +263,7 @@ class PlotApi:
                 return df
 
     def data_for_parameter(self, ensemble_id: str, parameter_key: str) -> pd.DataFrame:
-        with ErtServer.session(project=self.ens_path) as client:
+        with create_ertserver_client(self.ens_path) as client:
             parameter = client.get(
                 f"/ensembles/{ensemble_id}/parameters/{PlotApi.escape(parameter_key)}",
                 headers={"accept": "application/x-parquet"},
@@ -294,11 +301,12 @@ class PlotApi:
             )
             if not key_def:
                 raise httpx.RequestError(f"Response key {key_def} not found")
-
-            assert key_def.response_metadata is not None
-            actual_response_key = key_def.response_metadata.response_key
+            assert key_def.response is not None
+            actual_response_key = key
+            if "@" in actual_response_key:
+                actual_response_key = key.split("@", maxsplit=1)[0]
             filter_on = key_def.filter_on
-            with ErtServer.session(project=self.ens_path) as client:
+            with create_ertserver_client(self.ens_path) as client:
                 response = client.get(
                     f"/ensembles/{ensemble.id}/responses/{PlotApi.escape(actual_response_key)}/observations",
                     timeout=self._timeout,
@@ -386,7 +394,7 @@ class PlotApi:
         if not ensemble:
             return np.array([])
 
-        with ErtServer.session(project=self.ens_path) as client:
+        with create_ertserver_client(self.ens_path) as client:
             response = client.get(
                 f"/ensembles/{ensemble.id}/parameters/{PlotApi.escape(key)}/std_dev",
                 params={"z": z},

ert/gui/tools/plot/plot_widget.py

@@ -153,6 +153,7 @@ class PlotWidget(QWidget):
         # only for histogram plot see _sync_log_checkbox
         self._log_checkbox.setVisible(False)
         self._log_checkbox.setToolTip("Toggle data domain to log scale and back")
+        self._log_checkbox.clicked.connect(self.logLogScaleButtonUsage)
 
         log_checkbox_row = QHBoxLayout()
         log_checkbox_row.addWidget(self._log_checkbox)
@@ -193,6 +194,10 @@ class PlotWidget(QWidget):
     def name(self) -> str:
         return self._name
 
+    def logLogScaleButtonUsage(self) -> None:
+        logger.info(f"Plotwidget utility used: 'Log scale button' in tab '{self.name}'")
+        self._log_checkbox.clicked.disconnect()  # Log only once
+
     def updatePlot(
         self,
         plot_context: "PlotContext",

ert/gui/tools/plot/plot_window.py

@@ -25,7 +25,7 @@ from PyQt6.QtWidgets import (
 from ert.config.field import Field
 from ert.dark_storage.common import get_storage_api_version
 from ert.gui.ertwidgets import CopyButton, showWaitCursorWhileWaiting
-from ert.services._base_service import ServerBootFail
+from ert.services import ServerBootFail
 from ert.utils import log_duration
 
 from .customize import PlotCustomizer
@@ -267,10 +267,10 @@ class PlotWindow(QMainWindow):
             ensemble_to_data_map: dict[EnsembleObject, pd.DataFrame] = {}
             for ensemble in selected_ensembles:
                 try:
-                    if key_def.response_metadata is not None:
+                    if key_def.response is not None:
                         ensemble_to_data_map[ensemble] = self._api.data_for_response(
                             ensemble_id=ensemble.id,
-                            response_key=key_def.response_metadata.response_key,
+                            response_key=key,
                             filter_on=key_def.filter_on,
                         )
                     elif (
@@ -348,10 +348,7 @@ class PlotWindow(QMainWindow):
                     handle_exception(e)
                     plot_context.history_data = None
 
-            if (
-                key_def.response_metadata is not None
-                and key_def.response_metadata.response_type == "rft"
-            ):
+            if key_def.response is not None and key_def.response.type == "rft":
                 plot_context.setXLabel(key.split(":")[-1])
                 plot_context.setYLabel("TVD")
                 plot_context.depth_y_axis = True

ert/run_models/ensemble_experiment.py

@@ -1,16 +1,14 @@
 from __future__ import annotations
 
 import logging
-from typing import ClassVar, cast
+from typing import ClassVar
 from uuid import UUID
 
 from pydantic import PrivateAttr
 
 from ert.config import (
-    ParameterConfig,
     PostExperimentFixtures,
     PreExperimentFixtures,
-    ResponseConfig,
 )
 from ert.ensemble_evaluator import EvaluatorServerConfig
 from ert.run_models.initial_ensemble_run_model import (
@@ -54,13 +52,8 @@ class EnsembleExperiment(InitialEnsembleRunModel, EnsembleExperimentConfig):
         self.run_workflows(fixtures=PreExperimentFixtures(random_seed=self.random_seed))
 
         experiment_storage = self._storage.create_experiment(
-            parameters=cast(list[ParameterConfig], self.parameter_configuration),
-            observations={k: v.to_polars() for k, v in self.observations.items()}
-            if self.observations is not None
-            else None,
-            responses=cast(list[ResponseConfig], self.response_configuration),
+            experiment_config=self.model_dump(mode="json"),
             name=self.experiment_name,
-            templates=self.ert_templates,
         )
 
         ensemble_storage = self._storage.create_ensemble(

ert/run_models/ensemble_smoother.py

@@ -2,16 +2,13 @@ from __future__ import annotations
 
 import functools
 import logging
-from typing import cast
 
 import numpy as np
 from pydantic import PrivateAttr
 
 from ert.config import (
-    ParameterConfig,
     PostExperimentFixtures,
     PreExperimentFixtures,
-    ResponseConfig,
 )
 from ert.ensemble_evaluator import EvaluatorServerConfig
 from ert.run_models.initial_ensemble_run_model import (
@@ -48,13 +45,8 @@ class EnsembleSmoother(InitialEnsembleRunModel, UpdateRunModel, EnsembleSmoother
         self.run_workflows(fixtures=PreExperimentFixtures(random_seed=self.random_seed))
 
         experiment_storage = self._storage.create_experiment(
-            parameters=cast(list[ParameterConfig], self.parameter_configuration),
-            observations={k: v.to_polars() for k, v in self.observations.items()}
-            if self.observations is not None
-            else None,
-            responses=cast(list[ResponseConfig], self.response_configuration),
+            experiment_config=self.model_dump(mode="json"),
             name=self.experiment_name,
-            templates=self.ert_templates,
         )
 
         prior = self._storage.create_ensemble(

ert/run_models/everest_run_model.py

@@ -17,11 +17,11 @@ from enum import IntEnum, auto
 from functools import cached_property
 from pathlib import Path
 from types import TracebackType
-from typing import TYPE_CHECKING, Any, Protocol, cast
+from typing import TYPE_CHECKING, Annotated, Any, Protocol
 
 import numpy as np
 from numpy.typing import NDArray
-from pydantic import PrivateAttr, ValidationError
+from pydantic import Field, PrivateAttr, TypeAdapter, ValidationError
 from ropt.enums import ExitCode as RoptExitCode
 from ropt.evaluator import EvaluatorContext, EvaluatorResult
 from ropt.results import FunctionResults, Results
@@ -36,7 +36,6 @@ from ert.config import (
     GenDataConfig,
     HookRuntime,
     KnownQueueOptionsAdapter,
-    ParameterConfig,
     QueueConfig,
     ResponseConfig,
     SummaryConfig,
@@ -222,12 +221,24 @@ def _get_workflows(
     return res_hooks, res_workflows
 
 
+EverestResponseTypes = (
+    EverestObjectivesConfig | EverestConstraintsConfig | SummaryConfig | GenDataConfig
+)
+
+EverestResponseTypesAdapter = TypeAdapter(  # type: ignore
+    Annotated[
+        EverestResponseTypes,
+        Field(discriminator="type"),
+    ]
+)
+
+
 class EverestRunModelConfig(RunModelConfig):
     optimization_output_dir: str
     simulation_dir: str
 
-    parameter_configuration: list[ParameterConfig]
-    response_configuration: list[ResponseConfig]
+    parameter_configuration: list[EverestControl]
+    response_configuration: list[EverestResponseTypes]
 
     input_constraints: list[InputConstraintConfig]
     optimization: OptimizationConfig
@@ -542,7 +553,7 @@ class EverestRunModel(RunModel, EverestRunModelConfig):
 
         # There will and must always be one EverestControl config for an
         # Everest optimization.
-        return cast(list[EverestControl], controls)
+        return controls
 
     @cached_property
     def _transforms(self) -> EverestOptModelTransforms:
@@ -676,9 +687,7 @@ class EverestRunModel(RunModel, EverestRunModelConfig):
         self._eval_server_cfg = evaluator_server_config
 
         self._experiment = self._experiment or self._storage.create_experiment(
-            name=self.experiment_name,
-            parameters=self.parameter_configuration,
-            responses=self.response_configuration,
+            name=self.experiment_name, experiment_config=self.model_dump(mode="json")
         )
 
         self._experiment.status = ExperimentStatus(
@@ -764,7 +773,7 @@ class EverestRunModel(RunModel, EverestRunModelConfig):
 
     def _create_optimizer(self) -> tuple[BasicOptimizer, list[float]]:
         enopt_config, initial_guesses = everest2ropt(
-            cast(list[EverestControl], self.parameter_configuration),
+            self.parameter_configuration,
             self.objectives_config,
             self.input_constraints,
             self.output_constraints_config,
@@ -831,19 +840,18 @@ class EverestRunModel(RunModel, EverestRunModelConfig):
 
         ensemble.save_everest_realization_info(realization_info)
 
-        for sim_id in range(sim_to_control_vector.shape[0]):
-            sim_controls = sim_to_control_vector[sim_id]
-            offset = 0
-            for control_config in self._everest_control_configs:
-                n_param_keys = len(control_config.parameter_keys)
-
-                # Save controls to ensemble
-                ensemble.save_parameters_numpy(
-                    sim_controls[offset : (offset + n_param_keys)].reshape(-1, 1),
-                    control_config.name,
-                    np.array([sim_id]),
-                )
-                offset += n_param_keys
+        iens = sim_to_control_vector.shape[0]
+        offset = 0
+        for control_config in self._everest_control_configs:
+            n_param_keys = len(control_config.parameter_keys)
+            name = control_config.name
+            parameters = sim_to_control_vector[:, offset : offset + n_param_keys]
+            ensemble.save_parameters_numpy(
+                parameters.reshape(-1, n_param_keys),
+                name,
+                np.arange(iens),
+            )
+            offset += n_param_keys
 
         # Evaluate the batch:
         run_args = self._get_run_args(

ert/run_models/initial_ensemble_run_model.py

@@ -3,15 +3,17 @@ from typing import Annotated, Any, Literal, Self
 import numpy as np
 import polars as pl
 from polars.datatypes import DataTypeClass
-from pydantic import BaseModel, Field, field_validator
+from pydantic import BaseModel, Field
 
 from ert.config import (
     EverestControl,
     GenKwConfig,
     KnownResponseTypes,
+    Observation,
     SurfaceConfig,
 )
 from ert.config import Field as FieldConfig
+from ert.config._create_observation_dataframes import create_observation_dataframes
 from ert.ensemble_evaluator.config import EvaluatorServerConfig
 from ert.run_arg import create_run_arguments
 from ert.run_models.run_model import RunModel, RunModelConfig
@@ -67,27 +69,7 @@ class InitialEnsembleRunModelConfig(RunModelConfig):
         ]
     ]
     ert_templates: list[tuple[str, str]]
-    observations: dict[str, DictEncodedDataFrame] | None = None
-
-    @field_validator("observations", mode="before")
-    @classmethod
-    def make_dict_encoded_observations(
-        cls, v: dict[str, pl.DataFrame | DictEncodedDataFrame | dict[str, Any]] | None
-    ) -> dict[str, DictEncodedDataFrame] | None:
-        if v is None:
-            return None
-
-        encoded = {}
-        for k, df in v.items():
-            match df:
-                case DictEncodedDataFrame():
-                    encoded[k] = df
-                case pl.DataFrame():
-                    encoded[k] = DictEncodedDataFrame.from_polars(df)
-                case dict():
-                    encoded[k] = DictEncodedDataFrame.model_validate(df)
-
-        return encoded
+    observations: list[Observation] | None = None
 
 
 class InitialEnsembleRunModel(RunModel, InitialEnsembleRunModelConfig):
@@ -101,6 +83,7 @@ class InitialEnsembleRunModel(RunModel, InitialEnsembleRunModelConfig):
             np.where(self.active_realizations)[0],
             parameters=[param.name for param in self.parameter_configuration],
             random_seed=self.random_seed,
+            num_realizations=self.runpath_config.num_realizations,
             design_matrix_df=self.design_matrix.to_polars()
             if self.design_matrix is not None
             else None,
@@ -117,3 +100,17 @@ class InitialEnsembleRunModel(RunModel, InitialEnsembleRunModelConfig):
             evaluator_server_config,
         )
         return ensemble_storage
+
+    def observation_dataframes(self) -> dict[str, pl.DataFrame]:
+        if self.observations is None:
+            return {}
+
+        rft_config = next(
+            (r for r in self.response_configuration if r.type == "rft"),
+            None,
+        )
+
+        return create_observation_dataframes(
+            observations=self.observations,
+            rft_config=rft_config,
+        )

ert/run_models/manual_update.py

@@ -46,13 +46,19 @@ class ManualUpdate(UpdateRunModel, ManualUpdateConfig):
         self.set_env_key("_ERT_EXPERIMENT_ID", str(prior_experiment.id))
         self.set_env_key("_ERT_ENSEMBLE_ID", str(self._prior.id))
 
+        experiment_config = self.model_dump(mode="json") | {
+            "parameter_configuration": prior_experiment.experiment_config[
+                "parameter_configuration"
+            ],
+            "response_configuration": prior_experiment.experiment_config[
+                "response_configuration"
+            ],
+            "observations": prior_experiment.experiment_config["observations"],
+        }
+
         target_experiment = self._storage.create_experiment(
-            parameters=list(prior_experiment.parameter_configuration.values()),
-            responses=list(prior_experiment.response_configuration.values()),
-            observations=prior_experiment.observations,
-            simulation_arguments=prior_experiment.metadata,
+            experiment_config=experiment_config,
             name=f"Manual update of {self._prior.name}",
-            templates=self.ert_templates,
         )
         self.update(
             self._prior,

ert/run_models/model_factory.py

@@ -153,7 +153,7 @@ def _setup_single_test_run(
         log_path=config.analysis_config.log_path,
         storage_path=config.ens_path,
         queue_config=config.queue_config.create_local_copy(),
-        observations=config.observations,
+        observations=config.observation_declarations,
     )
 
     return SingleTestRun(
@@ -212,7 +212,7 @@ def _setup_ensemble_experiment(
         log_path=config.analysis_config.log_path,
         storage_path=config.ens_path,
         queue_config=config.queue_config,
-        observations=config.observations,
+        observations=config.observation_declarations,
     )
 
     return EnsembleExperiment(
@@ -305,7 +305,7 @@ def _setup_manual_update(
         hooked_workflows=config.hooked_workflows,
         log_path=config.analysis_config.log_path,
         ert_templates=config.ert_templates,
-        observations=config.observations,
+        observations=config.observation_declarations,
     )
     return ManualUpdate(**runmodel_config.model_dump(), status_queue=status_queue)
 
@@ -343,7 +343,7 @@ def _setup_manual_update_enif(
         substitutions=config.substitutions,
         hooked_workflows=config.hooked_workflows,
         log_path=config.analysis_config.log_path,
-        observations=config.observations,
+        observations=config.observation_declarations,
     )
 
 
@@ -389,7 +389,7 @@ def _setup_ensemble_smoother(
         substitutions=config.substitutions,
         hooked_workflows=config.hooked_workflows,
         log_path=config.analysis_config.log_path,
-        observations=config.observations,
+        observations=config.observation_declarations,
     )
     return EnsembleSmoother(**runmodel_config.model_dump(), status_queue=status_queue)
 
@@ -435,7 +435,7 @@ def _setup_ensemble_information_filter(
         substitutions=config.substitutions,
         hooked_workflows=config.hooked_workflows,
         log_path=config.analysis_config.log_path,
-        observations=config.observations,
+        observations=config.observation_declarations,
     )
     return EnsembleInformationFilter(
         **runmodel_config.model_dump(), status_queue=status_queue
@@ -507,7 +507,7 @@ def _setup_multiple_data_assimilation(
         substitutions=config.substitutions,
         hooked_workflows=config.hooked_workflows,
         log_path=config.analysis_config.log_path,
-        observations=config.observations,
+        observations=config.observation_declarations,
     )
     return MultipleDataAssimilation(
         **runmodel_config.model_dump(), status_queue=status_queue

ert/run_models/multiple_data_assimilation.py

@@ -2,16 +2,14 @@ from __future__ import annotations
 
 import functools
 import logging
-from typing import Any, ClassVar, cast
+from typing import Any, ClassVar
 from uuid import UUID
 
 from pydantic import PrivateAttr
 
 from ert.config import (
-    ParameterConfig,
     PostExperimentFixtures,
     PreExperimentFixtures,
-    ResponseConfig,
 )
 from ert.ensemble_evaluator import EvaluatorServerConfig
 from ert.run_models.initial_ensemble_run_model import (
@@ -97,12 +95,8 @@ class MultipleDataAssimilation(
                         f"restart iteration = {prior.iteration + 1}"
                     )
                 target_experiment = self._storage.create_experiment(
-                    parameters=list(prior.experiment.parameter_configuration.values()),
-                    responses=list(prior.experiment.response_configuration.values()),
-                    observations=prior.experiment.observations,
-                    simulation_arguments=prior.experiment.metadata,
+                    experiment_config=self.model_dump(mode="json"),
                     name=f"Restart from {prior.name}",
-                    templates=self.ert_templates,
                 )
 
             except (KeyError, ValueError) as err:
@@ -113,16 +107,9 @@ class MultipleDataAssimilation(
             self.run_workflows(
                 fixtures=PreExperimentFixtures(random_seed=self.random_seed),
             )
-            sim_args = {"weights": self.weights}
             experiment_storage = self._storage.create_experiment(
-                parameters=cast(list[ParameterConfig], self.parameter_configuration),
-                observations={k: v.to_polars() for k, v in self.observations.items()}
-                if self.observations is not None
-                else None,
-                responses=cast(list[ResponseConfig], self.response_configuration),
+                experiment_config=self.model_dump(mode="json"),
                 name=self.experiment_name,
-                templates=self.ert_templates,
-                simulation_arguments=sim_args,
             )
 
             prior = self._storage.create_ensemble(

ert/sample_prior.py

@@ -21,6 +21,7 @@ def sample_prior(
     ensemble: Ensemble,
     active_realizations: Iterable[int],
     random_seed: int,
+    num_realizations: int,
     parameters: list[str] | None = None,
     design_matrix_df: pl.DataFrame | None = None,
 ) -> None:
@@ -65,21 +66,18 @@ def sample_prior(
                     f"Sampling parameter {config_node.name} "
                     f"for realizations {active_realizations}"
                 )
-                datasets = [
-                    Ensemble.sample_parameter(
-                        config_node,
-                        realization_nr,
-                        random_seed=random_seed,
-                    )
-                    for realization_nr in active_realizations
-                ]
-                if datasets:
-                    dataset = pl.concat(datasets, how="vertical")
+                dataset = Ensemble.sample_parameter(
+                    config_node,
+                    list(active_realizations),
+                    random_seed=random_seed,
+                    num_realizations=num_realizations,
+                )
+            if not (dataset is None or dataset.is_empty()):
+                if complete_dataset is None:
+                    complete_dataset = dataset
+                elif dataset is not None:
+                    complete_dataset = complete_dataset.join(dataset, on="realization")
 
-            if complete_dataset is None:
-                complete_dataset = dataset
-            elif dataset is not None:
-                complete_dataset = complete_dataset.join(dataset, on="realization")
         else:
             for realization_nr in active_realizations:
                 ds = config_node.read_from_runpath(Path(), realization_nr, 0)