ert 18.0.9__py3-none-any.whl → 19.0.0__py3-none-any.whl

ert/run_models/everest_run_model.py

@@ -31,8 +31,8 @@ from typing_extensions import TypedDict
 
 from ert.config import (
     EverestConstraintsConfig,
+    EverestControl,
     EverestObjectivesConfig,
-    ExtParamConfig,
     GenDataConfig,
     HookRuntime,
     KnownQueueOptionsAdapter,
@@ -535,19 +535,19 @@ class EverestRunModel(RunModel, EverestRunModelConfig):
         )
 
     @property
-    def ext_param_configs(self) -> list[ExtParamConfig]:
-        ext_params = [
+    def _everest_control_configs(self) -> list[EverestControl]:
+        controls = [
             c for c in self.parameter_configuration if c.type == "everest_parameters"
         ]
 
-        # There will and must always be one extparam config for an
+        # There will and must always be one EverestControl config for an
         # Everest optimization.
-        return cast(list[ExtParamConfig], ext_params)
+        return cast(list[EverestControl], controls)
 
     @cached_property
     def _transforms(self) -> EverestOptModelTransforms:
         return get_optimization_domain_transforms(
-            self.ext_param_configs,
+            self._everest_control_configs,
             self.objectives_config,
             self.input_constraints,
             self.output_constraints_config,
@@ -693,7 +693,9 @@ class EverestRunModel(RunModel, EverestRunModelConfig):
         )
 
         formatted_control_names = [
-            name for config in self.ext_param_configs for name in config.input_keys
+            name
+            for config in self._everest_control_configs
+            for name in config.input_keys
         ]
         self._ever_storage.init(
             formatted_control_names=formatted_control_names,
@@ -762,7 +764,7 @@ class EverestRunModel(RunModel, EverestRunModelConfig):
 
     def _create_optimizer(self) -> tuple[BasicOptimizer, list[float]]:
         enopt_config, initial_guesses = everest2ropt(
-            cast(list[ExtParamConfig], self.parameter_configuration),
+            cast(list[EverestControl], self.parameter_configuration),
             self.objectives_config,
             self.input_constraints,
             self.output_constraints_config,
@@ -832,13 +834,13 @@ class EverestRunModel(RunModel, EverestRunModelConfig):
         for sim_id in range(sim_to_control_vector.shape[0]):
             sim_controls = sim_to_control_vector[sim_id]
             offset = 0
-            for ext_param_config in self.ext_param_configs:
-                n_param_keys = len(ext_param_config.parameter_keys)
+            for control_config in self._everest_control_configs:
+                n_param_keys = len(control_config.parameter_keys)
 
                 # Save controls to ensemble
                 ensemble.save_parameters_numpy(
                     sim_controls[offset : (offset + n_param_keys)].reshape(-1, 1),
-                    ext_param_config.name,
+                    control_config.name,
                     np.array([sim_id]),
                 )
                 offset += n_param_keys

ert/run_models/initial_ensemble_run_model.py

@@ -6,7 +6,7 @@ from polars.datatypes import DataTypeClass
 from pydantic import BaseModel, Field, field_validator
 
 from ert.config import (
-    ExtParamConfig,
+    EverestControl,
     GenKwConfig,
     KnownResponseTypes,
     SurfaceConfig,
@@ -56,7 +56,7 @@ class InitialEnsembleRunModelConfig(RunModelConfig):
     design_matrix: DictEncodedDataFrame | None
     parameter_configuration: list[
         Annotated[
-            (GenKwConfig | SurfaceConfig | FieldConfig | ExtParamConfig),
+            (GenKwConfig | SurfaceConfig | FieldConfig | EverestControl),
             Field(discriminator="type"),
         ]
     ]

ert/run_models/run_model.py

@@ -63,6 +63,7 @@ from ert.mode_definitions import MODULE_MODE
 from ert.runpaths import Runpaths
 from ert.storage import (
     Ensemble,
+    LocalStorage,
     Storage,
     open_storage,
 )
@@ -208,6 +209,10 @@ class RunModel(RunModelConfig, ABC):
     def model_post_init(self, ctx: Any) -> None:
         self._initial_realizations_mask = self.active_realizations.copy()
         self._completed_realizations_mask = [False] * len(self.active_realizations)
+
+        if LocalStorage.check_migration_needed(Path(self.storage_path)):
+            LocalStorage.perform_migration(Path(self.storage_path))
+
         self._storage = open_storage(self.storage_path, mode="w")
         self._rng = np.random.default_rng(self.random_seed)
         self._start_iteration = self.start_iteration
@@ -813,6 +818,10 @@ class RunModel(RunModelConfig, ABC):
         if self._max_parallelism_violation.amount > 0 and isinstance(
             self._max_parallelism_violation.message, str
         ):
+            logger.info(
+                "Warning displayed to user for NUM_CPU misconfiguration:\n"
+                f"{self._max_parallelism_violation.message}"
+            )
             warnings.warn(
                 self._max_parallelism_violation.message,
                 PostSimulationWarning,

ert/services/_base_service.py

@@ -12,6 +12,7 @@ import os
 import signal
 import sys
 import threading
+import types
 from collections.abc import Callable, Mapping, Sequence
 from logging import Logger, getLogger
 from pathlib import Path
@@ -23,7 +24,7 @@ from types import FrameType
 from typing import TYPE_CHECKING, Any, Generic, Self, TypedDict, TypeVar
 
 if TYPE_CHECKING:
-    from inspect import Traceback
+    pass
 
 T = TypeVar("T", bound="BaseService")
 
@@ -87,9 +88,9 @@ class _Context(Generic[T]):
 
     def __exit__(
         self,
-        exc_type: type[BaseException],
-        exc_value: BaseException,
-        traceback: Traceback,
+        exc_type: type[BaseException] | None,
+        exc_value: BaseException | None,
+        traceback: types.TracebackType | None,
     ) -> bool:
         self._service.shutdown()
         return exc_type is None
@@ -277,7 +278,7 @@ class BaseService:
             )
 
     @classmethod
-    def start_server(cls: type[T], *args: Any, **kwargs: Any) -> _Context[T]:
+    def start_server(cls, *args: Any, **kwargs: Any) -> _Context[Self]:
         if cls._instance is not None:
             raise RuntimeError("Server already running")
         cls._instance = obj = cls(*args, **kwargs)

ert/services/ert_server.py

@@ -5,8 +5,8 @@ import logging
 import os
 import sys
 import threading
+import types
 from collections.abc import Mapping
-from inspect import Traceback
 from pathlib import Path
 from tempfile import NamedTemporaryFile
 from time import sleep
@@ -28,9 +28,9 @@ class ErtServerContext:
 
     def __exit__(
         self,
-        exc_type: type[BaseException],
-        exc_value: BaseException,
-        traceback: Traceback,
+        exc_type: type[BaseException] | None,
+        exc_value: BaseException | None,
+        traceback: types.TracebackType | None,
     ) -> bool:
         self._service.shutdown()
         return exc_type is None

ert/shared/_doc_utils/__init__.py

@@ -8,12 +8,14 @@ from docutils.statemachine import StringList
 from sphinx.util.nodes import nested_parse_with_titles
 
 
-def _parse_raw_rst(rst_string: str, node: nodes.Node, state: Any) -> None:
+def _parse_raw_rst(rst_string: str, node: nodes.Element, state: Any) -> None:
     string_list = docutils.statemachine.StringList(list(rst_string.split("\n")))
     _parse_string_list(string_list, node, state)
 
 
-def _parse_string_list(string_list: StringList, node: nodes.Node, state: Any) -> None:
+def _parse_string_list(
+    string_list: StringList, node: nodes.Element, state: Any
+) -> None:
     nested_parse_with_titles(state, string_list, node)
 
 

ert/shared/net_utils.py

@@ -1,8 +1,10 @@
+import ipaddress
 import logging
 import random
 import socket
 from functools import lru_cache
 
+import psutil
 from dns import exception, resolver, reversename
 
 
@@ -50,6 +52,7 @@ def get_machine_name() -> str:
 def find_available_socket(
     host: str | None = None,
     port_range: range = range(51820, 51840 + 1),
+    prioritize_private_ip_address: bool = False,
 ) -> socket.socket:
     """
     The default and recommended approach here is to return a bound socket to the
@@ -71,7 +74,9 @@ def find_available_socket(
 
     See e.g. implementation and comments in EvaluatorServerConfig
     """
-    current_host = host if host is not None else get_ip_address()
+    current_host = (
+        host if host is not None else get_ip_address(prioritize_private_ip_address)
+    )
 
     if port_range.start == port_range.stop:
         ports = list(range(port_range.start, port_range.stop + 1))
@@ -135,20 +140,40 @@ def get_family(host: str) -> socket.AddressFamily:
         return socket.AF_INET6
 
 
-# See https://stackoverflow.com/a/28950776
-def get_ip_address() -> str:
-    try:
-        s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
-        try:
-            s.settimeout(0)
-            # try pinging a reserved, internal address in order
-            # to determine IP representing the default route
-            s.connect(("10.255.255.255", 1))
-            address = s.getsockname()[0]
-        finally:
-            s.close()
-    except BaseException:
-        logger.warning("Cannot determine ip-address. Falling back to localhost.")
-        address = "127.0.0.1"
-    logger.debug(f"ip-address: {address}")
-    return address
+def get_ip_address(prioritize_private: bool = False) -> str:
+    """
+    Get the first (private or public) IPv4 address of the current machine on the LAN.
+    Default behaviour returns the first public IP if found, then private, then loopback.
+
+    Parameters:
+        prioritize_private (bool): If True, private IP addresses are prioritized
+
+    Returns:
+        str: The selected IP address as a string.
+    """
+    loopback = ""
+    public = ""
+    private = ""
+    interfaces = psutil.net_if_addrs()
+    for addresses in interfaces.values():
+        for address in addresses:
+            if address.family.name == "AF_INET":
+                ip = address.address
+                if ipaddress.ip_address(ip).is_loopback and not loopback:
+                    loopback = ip
+                elif ipaddress.ip_address(ip).is_private and not private:
+                    private = ip
+                elif not public:
+                    public = ip
+
+    # Select first non-empty value, based on prioritization
+    if prioritize_private:
+        selected_ip = private or public or loopback
+    else:
+        selected_ip = public or private or loopback
+
+    if selected_ip:
+        return selected_ip
+    else:
+        logger.warning("Cannot determine ip-address. Falling back to 127.0.0.1")
+        return "127.0.0.1"

ert/shared/version.py

@@ -28,7 +28,7 @@ version_tuple: VERSION_TUPLE
 commit_id: COMMIT_ID
 __commit_id__: COMMIT_ID
 
-__version__ = version = '18.0.9'
-__version_tuple__ = version_tuple = (18, 0, 9)
+__version__ = version = '19.0.0'
+__version_tuple__ = version_tuple = (19, 0, 0)
 
-__commit_id__ = commit_id = 'ga202455d4'
+__commit_id__ = commit_id = 'g2518c4485'

ert/storage/__init__.py

@@ -39,6 +39,8 @@ class ErtStoragePermissionError(ErtStorageException):
 def open_storage(
     path: str | os.PathLike[str], mode: ModeLiteral | Mode = "r"
 ) -> Storage:
+    _ = LocalStorage.check_migration_needed(Path(path))
+
     try:
         return LocalStorage(Path(path), Mode(mode))
     except PermissionError as err:

ert/storage/local_ensemble.py

@@ -562,12 +562,18 @@ class LocalEnsemble(BaseMode):
         df = pl.scan_parquet(group_path)
         return df
 
-    def _load_scalar_keys(
+    def load_scalar_keys(
         self,
-        keys: list[str],
+        keys: list[str] | None = None,
         realizations: int | npt.NDArray[np.int_] | None = None,
         transformed: bool = False,
     ) -> pl.DataFrame:
+        if keys is None:
+            keys = self.experiment.parameter_keys
+        elif set(keys) - set(self.experiment.parameter_keys):
+            missing = set(keys) - set(self.experiment.parameter_keys)
+            raise KeyError(f"Parameters not registered to the experiment: {missing}")
+
         df_lazy = self._load_parameters_lazy(SCALAR_FILENAME)
         df_lazy = df_lazy.select(["realization", *keys])
         if realizations is not None:
@@ -620,7 +626,7 @@ class LocalEnsemble(BaseMode):
         cardinality = next(cfg.cardinality for cfg in cfgs)
 
         if cardinality == ParameterCardinality.multiple_configs_per_ensemble_dataset:
-            return self._load_scalar_keys(
+            return self.load_scalar_keys(
                 [cfg.name for cfg in cfgs], realizations, transformed
             )
         return self._load_dataset(
@@ -647,7 +653,7 @@ class LocalEnsemble(BaseMode):
         ]
         if keys:
             return (
-                self._load_scalar_keys(keys, realizations)
+                self.load_scalar_keys(keys, realizations)
                 .drop("realization")
                 .to_numpy()
                 .T.copy()
@@ -673,6 +679,9 @@ class LocalEnsemble(BaseMode):
                 if complete_df is None:
                     complete_df = ds
                 else:
+                    complete_df = complete_df.drop(
+                        [c for c in ds.columns if c != "realization"], strict=False
+                    )
                     complete_df = (
                         complete_df.join(ds, on="realization", how="left")
                         .unique(subset=["realization"], keep="first")
@@ -947,7 +956,7 @@ class LocalEnsemble(BaseMode):
         data = self.load_parameters(parameter_group)
         if isinstance(data, pl.DataFrame):
             return data.drop("realization").std().to_numpy().reshape(-1)
-        return data.std("realizations")["values"].values
+        return data.std("realizations")["values"].to_numpy()
 
     def get_parameter_state(
         self, realization: int
@@ -1048,7 +1057,7 @@ class LocalEnsemble(BaseMode):
                                 pl.col(col).is_in(observed_values.implode())
                             )
 
-                    pivoted = responses.collect(engine="streaming").pivot(
+                    pivoted = responses.collect(engine="streaming").pivot(  # noqa: PD010
                         on="realization",
                         index=["response_key", *response_cls.primary_key],
                         values="values",
@@ -1090,6 +1099,11 @@ class LocalEnsemble(BaseMode):
                             on=["response_key", *response_cls.primary_key],
                         )
 
+                    # Do not drop primary keys which
+                    # overlap with localization attributes
+                    primary_keys_to_drop = set(response_cls.primary_key).difference(
+                        {"north", "east", "radius"}
+                    )
                     joined = (
                         joined.with_columns(
                             pl.concat_str(
@@ -1099,7 +1113,7 @@ class LocalEnsemble(BaseMode):
                                 # Avoid potential collisions w/ primary key
                             )
                         )
-                        .drop(response_cls.primary_key)
+                        .drop(primary_keys_to_drop)
                         .rename({"__tmp_index_key__": "index"})
                     )
 
@@ -1115,6 +1129,9 @@ class LocalEnsemble(BaseMode):
                                 "observation_key",
                                 "observations",
                                 "std",
+                                "east",
+                                "north",
+                                "radius",
                             ]
                         )
 
@@ -1224,7 +1241,7 @@ class LocalEnsemble(BaseMode):
             how="horizontal",
         )
 
-        responses_wide = responses["realization", "response_key", "values"].pivot(
+        responses_wide = responses["realization", "response_key", "values"].pivot(  # noqa: PD010
             on="response_key", values="values"
         )
 

ert/storage/local_experiment.py

@@ -16,7 +16,7 @@ from pydantic import BaseModel, Field, TypeAdapter
 from surfio import IrapSurface
 
 from ert.config import (
-    ExtParamConfig,
+    EverestControl,
     GenKwConfig,
     KnownResponseTypes,
     ParameterConfig,
@@ -66,7 +66,7 @@ _responses_adapter = TypeAdapter(  # type: ignore
 
 _parameters_adapter = TypeAdapter(  # type: ignore
     Annotated[
-        (GenKwConfig | SurfaceConfig | FieldConfig | ExtParamConfig),
+        (GenKwConfig | SurfaceConfig | FieldConfig | EverestControl),
         Field(discriminator="type"),
     ]
 )

ert/storage/local_storage.py

@@ -5,14 +5,14 @@ import logging
 import os
 import re
 import shutil
+import types
 from collections.abc import Generator, MutableSequence
 from datetime import datetime
 from functools import cached_property
 from pathlib import Path
 from tempfile import NamedTemporaryFile
 from textwrap import dedent
-from types import TracebackType
-from typing import Any
+from typing import Any, Self
 from uuid import UUID, uuid4
 
 import polars as pl
@@ -20,6 +20,7 @@ import xarray as xr
 from filelock import FileLock, Timeout
 from pydantic import BaseModel, Field
 
+import ert.storage
 from ert.config import ErtConfig, ParameterConfig, ResponseConfig
 from ert.shared import __version__
 
@@ -30,7 +31,7 @@ from .realization_storage_state import RealizationStorageState
 
 logger = logging.getLogger(__name__)
 
-_LOCAL_STORAGE_VERSION = 18
+_LOCAL_STORAGE_VERSION = 23
 
 
 class _Migrations(BaseModel):
@@ -64,6 +65,7 @@ class LocalStorage(BaseMode):
         self,
         path: str | os.PathLike[str],
         mode: Mode,
+        stage_for_migration: bool = False,
     ) -> None:
         """
         Initializes the LocalStorage instance.
@@ -74,6 +76,8 @@ class LocalStorage(BaseMode):
             The file system path to the storage.
         mode : Mode
             The access mode for the storage (read/write).
+        stage_for_migration : bool
+            Whether to avoid reloading storage to allow migration
         """
 
         self.path = Path(path).absolute()
@@ -82,9 +86,9 @@ class LocalStorage(BaseMode):
         if mode.can_write:
             self._acquire_lock()
 
-        self._experiments: dict[UUID, LocalExperiment]
-        self._ensembles: dict[UUID, LocalEnsemble]
-        self._index: _Index
+        self._experiments: dict[UUID, LocalExperiment] = {}
+        self._ensembles: dict[UUID, LocalEnsemble] = {}
+        self._index: _Index = _Index()
 
         try:
             self.version = _storage_version(self.path)
@@ -101,19 +105,18 @@ class LocalStorage(BaseMode):
                 raise ValueError(f"No index.json, but found: {errors}") from err
             self.version = _LOCAL_STORAGE_VERSION
 
-        if self.check_migration_needed(Path(self.path)):
-            if not self.can_write:
-                raise RuntimeError(
-                    f"Cannot open storage '{self.path}' in read-only mode: "
-                    f"Storage version {self.version} is too old. "
-                    f"Run ert to initiate migration."
-                )
-            else:
-                self._migrate(self.version)
+        if self.check_migration_needed(Path(self.path)) and not self.can_write:
+            raise RuntimeError(
+                f"Cannot open storage '{self.path}' in read-only mode: "
+                f"Storage version {self.version} is too old. "
+                f"Run ert to initiate migration."
+            )
 
-        self.refresh()
-        if mode.can_write:
-            self._save_index()
+        if not stage_for_migration:
+            self.reload()
+
+            if mode.can_write:
+                self._save_index()
 
     @staticmethod
     def check_migration_needed(storage_dir: Path) -> bool:
@@ -123,7 +126,7 @@ class LocalStorage(BaseMode):
             version = _LOCAL_STORAGE_VERSION
 
         if version > _LOCAL_STORAGE_VERSION:
-            raise RuntimeError(
+            raise ert.storage.ErtStorageException(
                 f"Cannot open storage '{storage_dir.absolute()}': Storage version "
                 f"{version} is newer than the current version {_LOCAL_STORAGE_VERSION}"
                 f", upgrade ert to continue, or run with a different ENSPATH"
@@ -131,7 +134,14 @@ class LocalStorage(BaseMode):
 
         return version < _LOCAL_STORAGE_VERSION
 
-    def refresh(self) -> None:
+    @staticmethod
+    def perform_migration(path: Path) -> None:
+        if LocalStorage.check_migration_needed(path):
+            with LocalStorage(path, Mode("w"), True) as storage:
+                storage._migrate(storage.version)
+                storage.reload()
+
+    def reload(self) -> None:
         """
         Reloads the index, experiments, and ensembles from the storage.
 
@@ -267,14 +277,14 @@ class LocalStorage(BaseMode):
     def _swap_path(self) -> Path:
         return self.path / self.SWAP_PATH
 
-    def __enter__(self) -> LocalStorage:
+    def __enter__(self) -> Self:
         return self
 
     def __exit__(
         self,
-        exception: Exception,
-        exception_type: type[Exception],
-        traceback: TracebackType,
+        exception: type[BaseException] | None,
+        exception_type: BaseException | None,
+        traceback: types.TracebackType | None,
     ) -> None:
         self.close()
 
@@ -504,6 +514,11 @@ class LocalStorage(BaseMode):
             to16,
             to17,
             to18,
+            to19,
+            to20,
+            to21,
+            to22,
+            to23,
         )
 
         try:
@@ -533,7 +548,7 @@ class LocalStorage(BaseMode):
 
                 self._index = self._load_index()
 
-                logger.info("storage backed up for version less than 6")
+                logger.info("Storage backed up for version less than 5")
                 print(self._legacy_storage_migration_message(bkup_path, "14.6.*"))
                 return None
             elif version < _LOCAL_STORAGE_VERSION:
@@ -551,6 +566,11 @@ class LocalStorage(BaseMode):
                     15: to16,
                     16: to17,
                     17: to18,
+                    18: to19,
+                    19: to20,
+                    20: to21,
+                    21: to22,
+                    22: to23,
                 }
                 for from_version in range(version, _LOCAL_STORAGE_VERSION):
                     migrations[from_version].migrate(self.path)

ert/storage/migration/to11.py

@@ -44,7 +44,7 @@ def migrate(path: Path) -> None:
                         array = ds.isel(realizations=0, drop=True)["values"]
                         realization = int(real_dir.name.split("-")[1])
 
-                        def parse_value(value: float | int | str) -> float | int | str:
+                        def parse_value(value: float | str) -> float | int | str:
                             if isinstance(value, float | int):
                                 return value
                             try:

ert/storage/migration/to18.py

@@ -4,7 +4,6 @@ info = (
     "Added localization attributes to summary observations."
     "Added RFT observations."
     "No change to current storage, only additions. "
-    "Bumping to 17 to indicate appended schema in storage."
 )
 
 

ert/storage/migration/to19.py

@@ -0,0 +1,34 @@
+import json
+from pathlib import Path
+from typing import Any
+
+info = "Add dimensionality attribute to parameters"
+
+
+def migrate_param(parameters_json: dict[str, Any]) -> dict[str, Any]:
+    new_configs = {}
+    for param_config in parameters_json.values():
+        if param_config["type"] == "surface":
+            param_config["dimensionality"] = 2
+        elif param_config["type"] == "field":
+            param_config["dimensionality"] = 3
+        else:
+            param_config["dimensionality"] = 1
+
+        new_configs[param_config["name"]] = param_config
+    return new_configs
+
+
+def migrate_parameters_for_experiment(experiment: Path) -> None:
+    with open(experiment / "parameter.json", encoding="utf-8") as fin:
+        parameters_json = json.load(fin)
+
+    new_parameter_configs = migrate_param(parameters_json)
+    Path(experiment / "parameter.json").write_text(
+        json.dumps(new_parameter_configs, indent=2), encoding="utf-8"
+    )
+
+
+def migrate(path: Path) -> None:
+    for experiment in path.glob("experiments/*"):
+        migrate_parameters_for_experiment(experiment)