ert 20.0.0b0__py3-none-any.whl → 20.0.0b1__py3-none-any.whl

ert/run_models/ensemble_smoother.py

@@ -2,16 +2,13 @@ from __future__ import annotations
 
 import functools
 import logging
-from typing import cast
 
 import numpy as np
 from pydantic import PrivateAttr
 
 from ert.config import (
-    ParameterConfig,
     PostExperimentFixtures,
     PreExperimentFixtures,
-    ResponseConfig,
 )
 from ert.ensemble_evaluator import EvaluatorServerConfig
 from ert.run_models.initial_ensemble_run_model import (
@@ -48,11 +45,8 @@ class EnsembleSmoother(InitialEnsembleRunModel, UpdateRunModel, EnsembleSmoother
         self.run_workflows(fixtures=PreExperimentFixtures(random_seed=self.random_seed))
 
         experiment_storage = self._storage.create_experiment(
-            parameters=cast(list[ParameterConfig], self.parameter_configuration),
-            observations=self.observation_dataframes(),
-            responses=cast(list[ResponseConfig], self.response_configuration),
+            experiment_config=self.model_dump(mode="json"),
             name=self.experiment_name,
-            templates=self.ert_templates,
         )
 
         prior = self._storage.create_ensemble(

ert/run_models/everest_run_model.py

@@ -17,11 +17,11 @@ from enum import IntEnum, auto
 from functools import cached_property
 from pathlib import Path
 from types import TracebackType
-from typing import TYPE_CHECKING, Any, Protocol, cast
+from typing import TYPE_CHECKING, Annotated, Any, Protocol
 
 import numpy as np
 from numpy.typing import NDArray
-from pydantic import PrivateAttr, ValidationError
+from pydantic import Field, PrivateAttr, TypeAdapter, ValidationError
 from ropt.enums import ExitCode as RoptExitCode
 from ropt.evaluator import EvaluatorContext, EvaluatorResult
 from ropt.results import FunctionResults, Results
@@ -36,7 +36,6 @@ from ert.config import (
     GenDataConfig,
     HookRuntime,
     KnownQueueOptionsAdapter,
-    ParameterConfig,
     QueueConfig,
     ResponseConfig,
     SummaryConfig,
@@ -222,12 +221,24 @@ def _get_workflows(
     return res_hooks, res_workflows
 
 
+EverestResponseTypes = (
+    EverestObjectivesConfig | EverestConstraintsConfig | SummaryConfig | GenDataConfig
+)
+
+EverestResponseTypesAdapter = TypeAdapter(  # type: ignore
+    Annotated[
+        EverestResponseTypes,
+        Field(discriminator="type"),
+    ]
+)
+
+
 class EverestRunModelConfig(RunModelConfig):
     optimization_output_dir: str
     simulation_dir: str
 
-    parameter_configuration: list[ParameterConfig]
-    response_configuration: list[ResponseConfig]
+    parameter_configuration: list[EverestControl]
+    response_configuration: list[EverestResponseTypes]
 
     input_constraints: list[InputConstraintConfig]
     optimization: OptimizationConfig
@@ -542,7 +553,7 @@ class EverestRunModel(RunModel, EverestRunModelConfig):
 
         # There will and must always be one EverestControl config for an
         # Everest optimization.
-        return cast(list[EverestControl], controls)
+        return controls
 
     @cached_property
     def _transforms(self) -> EverestOptModelTransforms:
@@ -676,9 +687,7 @@ class EverestRunModel(RunModel, EverestRunModelConfig):
         self._eval_server_cfg = evaluator_server_config
 
         self._experiment = self._experiment or self._storage.create_experiment(
-            name=self.experiment_name,
-            parameters=self.parameter_configuration,
-            responses=self.response_configuration,
+            name=self.experiment_name, experiment_config=self.model_dump(mode="json")
         )
 
         self._experiment.status = ExperimentStatus(
@@ -764,7 +773,7 @@ class EverestRunModel(RunModel, EverestRunModelConfig):
 
     def _create_optimizer(self) -> tuple[BasicOptimizer, list[float]]:
         enopt_config, initial_guesses = everest2ropt(
-            cast(list[EverestControl], self.parameter_configuration),
+            self.parameter_configuration,
             self.objectives_config,
             self.input_constraints,
             self.output_constraints_config,

ert/run_models/manual_update.py

@@ -46,13 +46,19 @@ class ManualUpdate(UpdateRunModel, ManualUpdateConfig):
         self.set_env_key("_ERT_EXPERIMENT_ID", str(prior_experiment.id))
         self.set_env_key("_ERT_ENSEMBLE_ID", str(self._prior.id))
 
+        experiment_config = self.model_dump(mode="json") | {
+            "parameter_configuration": prior_experiment.experiment_config[
+                "parameter_configuration"
+            ],
+            "response_configuration": prior_experiment.experiment_config[
+                "response_configuration"
+            ],
+            "observations": prior_experiment.experiment_config["observations"],
+        }
+
         target_experiment = self._storage.create_experiment(
-            parameters=list(prior_experiment.parameter_configuration.values()),
-            responses=list(prior_experiment.response_configuration.values()),
-            observations=prior_experiment.observations,
-            simulation_arguments=prior_experiment.metadata,
+            experiment_config=experiment_config,
             name=f"Manual update of {self._prior.name}",
-            templates=self.ert_templates,
         )
         self.update(
             self._prior,

ert/run_models/multiple_data_assimilation.py

@@ -2,16 +2,14 @@ from __future__ import annotations
 
 import functools
 import logging
-from typing import Any, ClassVar, cast
+from typing import Any, ClassVar
 from uuid import UUID
 
 from pydantic import PrivateAttr
 
 from ert.config import (
-    ParameterConfig,
     PostExperimentFixtures,
     PreExperimentFixtures,
-    ResponseConfig,
 )
 from ert.ensemble_evaluator import EvaluatorServerConfig
 from ert.run_models.initial_ensemble_run_model import (
@@ -97,12 +95,8 @@ class MultipleDataAssimilation(
                         f"restart iteration = {prior.iteration + 1}"
                     )
                 target_experiment = self._storage.create_experiment(
-                    parameters=list(prior.experiment.parameter_configuration.values()),
-                    responses=list(prior.experiment.response_configuration.values()),
-                    observations=prior.experiment.observations,
-                    simulation_arguments=prior.experiment.metadata,
+                    experiment_config=self.model_dump(mode="json"),
                     name=f"Restart from {prior.name}",
-                    templates=self.ert_templates,
                 )
 
             except (KeyError, ValueError) as err:
@@ -113,14 +107,9 @@ class MultipleDataAssimilation(
             self.run_workflows(
                 fixtures=PreExperimentFixtures(random_seed=self.random_seed),
             )
-            sim_args = {"weights": self.weights}
             experiment_storage = self._storage.create_experiment(
-                parameters=cast(list[ParameterConfig], self.parameter_configuration),
-                observations=self.observation_dataframes(),
-                responses=cast(list[ResponseConfig], self.response_configuration),
+                experiment_config=self.model_dump(mode="json"),
                 name=self.experiment_name,
-                templates=self.ert_templates,
-                simulation_arguments=sim_args,
             )
 
             prior = self._storage.create_ensemble(

ert/scheduler/job.py

@@ -482,6 +482,7 @@ async def log_warnings_from_forward_model(
     real: Realization,
     job_submission_time: float,
     timeout_seconds: int = Job.DEFAULT_FILE_VERIFICATION_TIMEOUT,
+    max_logged_warnings: int = 200,
 ) -> int:
     """Parse all stdout and stderr files from running the forward model
     for anything that looks like a Warning, and log it.
@@ -513,11 +514,19 @@ async def log_warnings_from_forward_model(
         )
 
     async def log_warnings_from_file(
-        file: Path, iens: int, step: ForwardModelStep, step_idx: int, filetype: str
-    ) -> None:
+        file: Path,
+        iens: int,
+        step: ForwardModelStep,
+        step_idx: int,
+        filetype: str,
+        max_warnings_to_log: int,
+    ) -> int:
+        """Returns how many times a warning was logged"""
         captured: list[str] = []
         file_text = await anyio.Path(file).read_text(encoding="utf-8")
         for line in file_text.splitlines():
+            if len(captured) >= max_warnings_to_log:
+                break
             if line_contains_warning(line):
                 captured.append(line[:max_length])
 
@@ -528,6 +537,7 @@ async def log_warnings_from_forward_model(
             )
             warnings.warn(warning_msg, PostSimulationWarning, stacklevel=2)
             logger.warning(warning_msg)
+        return len(captured)
 
     async def wait_for_file(file_path: Path, _timeout: int) -> int:
         if _timeout <= 0:
@@ -546,6 +556,7 @@ async def log_warnings_from_forward_model(
                 break
         return remaining_timeout
 
+    log_count = 0
     with suppress(KeyError):
         runpath = Path(real.run_arg.runpath)
         for step_idx, step in enumerate(real.fm_steps):
@@ -560,9 +571,18 @@ async def log_warnings_from_forward_model(
                     if timeout_seconds <= 0:
                         break
 
-                    await log_warnings_from_file(
-                        std_path, real.iens, step, step_idx, file_type
+                    log_count += await log_warnings_from_file(
+                        std_path,
+                        real.iens,
+                        step,
+                        step_idx,
+                        file_type,
+                        max_logged_warnings - log_count,
                     )
             if timeout_seconds <= 0:
                 break
+    if log_count >= max_logged_warnings:
+        logger.warning(
+            "Reached maximum number of forward model step warnings to extract"
+        )
     return timeout_seconds

ert/shared/net_utils.py

@@ -1,10 +1,8 @@
-import ipaddress
 import logging
 import random
 import socket
 from functools import lru_cache
 
-import psutil
 from dns import exception, resolver, reversename
 
 
@@ -52,7 +50,6 @@ def get_machine_name() -> str:
 def find_available_socket(
     host: str | None = None,
     port_range: range = range(51820, 51840 + 1),
-    prioritize_private_ip_address: bool = False,
 ) -> socket.socket:
     """
     The default and recommended approach here is to return a bound socket to the
@@ -74,9 +71,7 @@ def find_available_socket(
 
     See e.g. implementation and comments in EvaluatorServerConfig
     """
-    current_host = (
-        host if host is not None else get_ip_address(prioritize_private_ip_address)
-    )
+    current_host = host if host is not None else get_ip_address()
 
     if port_range.start == port_range.stop:
         ports = list(range(port_range.start, port_range.stop + 1))
@@ -140,40 +135,20 @@ def get_family(host: str) -> socket.AddressFamily:
         return socket.AF_INET6
 
 
-def get_ip_address(prioritize_private: bool = False) -> str:
-    """
-    Get the first (private or public) IPv4 address of the current machine on the LAN.
-    Default behaviour returns the first public IP if found, then private, then loopback.
-
-    Parameters:
-        prioritize_private (bool): If True, private IP addresses are prioritized
-
-    Returns:
-        str: The selected IP address as a string.
-    """
-    loopback = ""
-    public = ""
-    private = ""
-    interfaces = psutil.net_if_addrs()
-    for addresses in interfaces.values():
-        for address in addresses:
-            if address.family.name == "AF_INET":
-                ip = address.address
-                if ipaddress.ip_address(ip).is_loopback and not loopback:
-                    loopback = ip
-                elif ipaddress.ip_address(ip).is_private and not private:
-                    private = ip
-                elif not public:
-                    public = ip
-
-    # Select first non-empty value, based on prioritization
-    if prioritize_private:
-        selected_ip = private or public or loopback
-    else:
-        selected_ip = public or private or loopback
-
-    if selected_ip:
-        return selected_ip
-    else:
-        logger.warning("Cannot determine ip-address. Falling back to 127.0.0.1")
-        return "127.0.0.1"
+# See https://stackoverflow.com/a/28950776
+def get_ip_address() -> str:
+    try:
+        s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+        try:
+            s.settimeout(0)
+            # try pinging a reserved, internal address in order
+            # to determine IP representing the default route
+            s.connect(("10.255.255.255", 1))
+            address = s.getsockname()[0]
+        finally:
+            s.close()
+    except BaseException:
+        logger.warning("Cannot determine ip-address. Falling back to localhost.")
+        address = "127.0.0.1"
+    logger.debug(f"ip-address: {address}")
+    return address

ert/shared/version.py

@@ -28,7 +28,7 @@ version_tuple: VERSION_TUPLE
 commit_id: COMMIT_ID
 __commit_id__: COMMIT_ID
 
-__version__ = version = '20.0.0b0'
-__version_tuple__ = version_tuple = (20, 0, 0, 'b0')
+__version__ = version = '20.0.0b1'
+__version_tuple__ = version_tuple = (20, 0, 0, 'b1')
 
-__commit_id__ = commit_id = 'g23469e640'
+__commit_id__ = commit_id = 'g4156859a8'

ert/storage/local_ensemble.py

@@ -1077,7 +1077,9 @@ class LocalEnsemble(BaseMode):
                     for col, observed_values in observed_cols.items():
                         if col != "time":
                             responses = responses.filter(
-                                pl.col(col).is_in(observed_values.implode())
+                                pl.col(col).is_in(
+                                    observed_values.implode(), nulls_equal=True
+                                )
                             )
 
                     pivoted = responses.collect(engine="streaming").pivot(
@@ -1120,6 +1122,7 @@ class LocalEnsemble(BaseMode):
                             pivoted,
                             how="left",
                             on=["response_key", *response_cls.primary_key],
+                            nulls_equal=True,
                         )
 
                     # Do not drop primary keys which

ert/storage/local_experiment.py

@@ -24,7 +24,10 @@ from ert.config import (
     SurfaceConfig,
 )
 from ert.config import Field as FieldConfig
-from ert.config.parsing.context_values import ContextBoolEncoder
+from ert.config._create_observation_dataframes import (
+    create_observation_dataframes,
+)
+from ert.config._observations import Observation
 
 from .mode import BaseMode, Mode, require_write
 
@@ -54,6 +57,7 @@ class _Index(BaseModel):
     # from a different experiment. For example, a manual update
     # is a separate experiment from the one that created the prior.
     ensembles: list[UUID]
+    experiment: dict[str, Any] = {}
     status: ExperimentStatus | None = Field(default=None)
 
 
@@ -80,9 +84,6 @@ class LocalExperiment(BaseMode):
     arguments. Provides methods to create and access associated ensembles.
     """
 
-    _parameter_file = Path("parameter.json")
-    _responses_file = Path("responses.json")
-    _metadata_file = Path("metadata.json")
     _templates_file = Path("templates.json")
     _index_file = Path("index.json")
 
@@ -118,63 +119,21 @@ class LocalExperiment(BaseMode):
         storage: LocalStorage,
         uuid: UUID,
         path: Path,
-        *,
-        parameters: list[ParameterConfig] | None = None,
-        responses: list[ResponseConfig] | None = None,
-        observations: dict[str, pl.DataFrame] | None = None,
-        simulation_arguments: dict[Any, Any] | None = None,
+        experiment_config: dict[str, Any],
         name: str | None = None,
-        templates: list[tuple[str, str]] | None = None,
     ) -> LocalExperiment:
-        """
-        Create a new LocalExperiment and store its configuration data.
-
-        Parameters
-        ----------
-        storage : LocalStorage
-            Storage instance for experiment creation.
-        uuid : UUID
-            Unique identifier for the new experiment.
-        path : Path
-            File system path for storing experiment data.
-        parameters : list of ParameterConfig, optional
-            List of parameter configurations.
-        responses : list of ResponseConfig, optional
-            List of response configurations.
-        observations : dict of str to encoded observation datasets, optional
-            Observations dictionary.
-        simulation_arguments : SimulationArguments, optional
-            Simulation arguments for the experiment.
-        name : str, optional
-            Experiment name. Defaults to current date if None.
-        templates : list of tuple[str, str], optional
-            Run templates for the experiment. Defaults to None.
-
-        Returns
-        -------
-        local_experiment : LocalExperiment
-            Instance of the newly created experiment.
-        """
         if name is None:
             name = datetime.today().isoformat()
 
         storage._write_transaction(
             path / cls._index_file,
-            _Index(id=uuid, name=name, ensembles=[])
+            _Index(id=uuid, name=name, ensembles=[], experiment=experiment_config)
             .model_dump_json(indent=2, exclude_none=True)
             .encode("utf-8"),
         )
 
-        parameter_data = {}
-        for parameter in parameters or []:
-            parameter.save_experiment_data(path)
-            parameter_data.update({parameter.name: parameter.model_dump(mode="json")})
-        storage._write_transaction(
-            path / cls._parameter_file,
-            json.dumps(parameter_data, indent=2).encode("utf-8"),
-        )
-
-        if templates:
+        templates = experiment_config.get("ert_templates")
+        if templates is not None:
             templates_path = path / "templates"
             templates_path.mkdir(parents=True, exist_ok=True)
             templates_abs: list[tuple[str, str]] = []
@@ -191,27 +150,29 @@ class LocalExperiment(BaseMode):
                 json.dumps(templates_abs).encode("utf-8"),
             )
 
-        response_data = {}
-        for response in responses or []:
-            response_data.update({response.type: response.model_dump(mode="json")})
-        storage._write_transaction(
-            path / cls._responses_file,
-            json.dumps(response_data, default=str, indent=2).encode("utf-8"),
-        )
-
-        if observations:
+        observation_declarations = experiment_config.get("observations")
+        if observation_declarations:
             output_path = path / "observations"
-            output_path.mkdir()
-            for response_type, dataset in observations.items():
-                storage._to_parquet_transaction(
-                    output_path / f"{response_type}", dataset
-                )
+            output_path.mkdir(parents=True, exist_ok=True)
 
-        simulation_data = simulation_arguments or {}
-        storage._write_transaction(
-            path / cls._metadata_file,
-            json.dumps(simulation_data, cls=ContextBoolEncoder).encode("utf-8"),
-        )
+            responses_list = experiment_config.get("response_configuration", [])
+            rft_config_json = next(
+                (r for r in responses_list if r.get("type") == "rft"), None
+            )
+            rft_config = (
+                _responses_adapter.validate_python(rft_config_json)
+                if rft_config_json is not None
+                else None
+            )
+
+            obs_adapter = TypeAdapter(Observation)  # type: ignore
+            obs_objs: list[Observation] = []
+            for od in observation_declarations:
+                obs_objs.append(obs_adapter.validate_python(od))
+
+            datasets = create_observation_dataframes(obs_objs, rft_config)
+            for response_type, df in datasets.items():
+                storage._to_parquet_transaction(output_path / response_type, df)
 
         return cls(storage, path, Mode.WRITE)
 
@@ -285,16 +246,10 @@ class LocalExperiment(BaseMode):
                 return ens
         raise KeyError(f"Ensemble with name '{name}' not found")
 
-    @property
-    def metadata(self) -> dict[str, Any]:
-        path = self.mount_point / self._metadata_file
-        if not path.exists():
-            raise ValueError(f"{self._metadata_file!s} does not exist")
-        return json.loads(path.read_text(encoding="utf-8"))
-
     @property
     def relative_weights(self) -> str:
-        return self.metadata.get("weights", "")
+        assert self.experiment_config is not None
+        return self.experiment_config.get("weights", "")
 
     @property
     def name(self) -> str:
@@ -324,9 +279,8 @@ class LocalExperiment(BaseMode):
 
     @property
     def parameter_info(self) -> dict[str, Any]:
-        return json.loads(
-            (self.mount_point / self._parameter_file).read_text(encoding="utf-8")
-        )
+        parameters_list = self.experiment_config.get("parameter_configuration", [])
+        return {parameter["name"]: parameter for parameter in parameters_list}
 
     @property
     def templates_configuration(self) -> list[tuple[str, str]]:
@@ -348,9 +302,8 @@ class LocalExperiment(BaseMode):
 
     @property
     def response_info(self) -> dict[str, Any]:
-        return json.loads(
-            (self.mount_point / self._responses_file).read_text(encoding="utf-8")
-        )
+        responses_list = self.experiment_config.get("response_configuration", [])
+        return {response["type"]: response for response in responses_list}
 
     def get_surface(self, name: str) -> IrapSurface:
         """
@@ -420,10 +373,50 @@ class LocalExperiment(BaseMode):
 
     @cached_property
     def observations(self) -> dict[str, pl.DataFrame]:
-        observations = sorted(self.mount_point.glob("observations/*"))
+        obs_dir = self.mount_point / "observations"
+
+        if obs_dir.exists():
+            datasets: dict[str, pl.DataFrame] = {}
+            for p in obs_dir.iterdir():
+                if not p.is_file():
+                    continue
+                try:
+                    df = pl.read_parquet(p)
+                except Exception:
+                    continue
+                datasets[p.stem] = df
+            return datasets
+
+        serialized_observations = self.experiment_config.get("observations", None)
+        if not serialized_observations:
+            return {}
+
+        output_path = self.mount_point / "observations"
+        output_path.mkdir(parents=True, exist_ok=True)
+
+        rft_cfg = None
+        try:
+            responses_list = self.experiment_config.get("response_configuration", [])
+            for r in responses_list:
+                if r.get("type") == "rft":
+                    rft_cfg = _responses_adapter.validate_python(r)
+                    break
+        except Exception:
+            rft_cfg = None
+
+        obs_adapter = TypeAdapter(Observation)  # type: ignore
+        obs_objs: list[Observation] = []
+        for od in serialized_observations:
+            obs_objs.append(obs_adapter.validate_python(od))
+
+        datasets = create_observation_dataframes(obs_objs, rft_cfg)
+        for response_type, df in datasets.items():
+            self._storage._to_parquet_transaction(output_path / response_type, df)
+
         return {
-            observation.name: pl.read_parquet(f"{observation}")
-            for observation in observations
+            p.stem: pl.read_parquet(p)
+            for p in (self.mount_point / "observations").iterdir()
+            if p.is_file()
         }
 
     @cached_property
@@ -489,18 +482,22 @@ class LocalExperiment(BaseMode):
             )
 
         config = responses_configuration[response_type]
+
         config.keys = sorted(response_keys)
         config.has_finalized_keys = True
+
+        response_index = next(
+            i
+            for i, c in enumerate(self.experiment_config["response_configuration"])
+            if c["type"] == response_type
+        )
+        self.experiment_config["response_configuration"][response_index] = (
+            config.model_dump(mode="json")
+        )
+
         self._storage._write_transaction(
-            self._path / self._responses_file,
-            json.dumps(
-                {
-                    c.type: c.model_dump(mode="json")
-                    for c in responses_configuration.values()
-                },
-                default=str,
-                indent=2,
-            ).encode("utf-8"),
+            self._path / self._index_file,
+            self._index.model_dump_json(indent=2).encode("utf-8"),
         )
 
         if self.response_key_to_response_type is not None:
@@ -508,3 +505,7 @@ class LocalExperiment(BaseMode):
 
         if self.response_type_to_response_keys is not None:
             del self.response_type_to_response_keys
+
+    @property
+    def experiment_config(self) -> dict[str, Any]:
+        return self._index.experiment