ert 18.0.9__py3-none-any.whl → 19.0.0rc0__py3-none-any.whl

ert/run_models/initial_ensemble_run_model.py

@@ -6,7 +6,7 @@ from polars.datatypes import DataTypeClass
 from pydantic import BaseModel, Field, field_validator
 
 from ert.config import (
-    ExtParamConfig,
+    EverestControl,
     GenKwConfig,
     KnownResponseTypes,
     SurfaceConfig,
@@ -56,7 +56,7 @@ class InitialEnsembleRunModelConfig(RunModelConfig):
     design_matrix: DictEncodedDataFrame | None
     parameter_configuration: list[
         Annotated[
-            (GenKwConfig | SurfaceConfig | FieldConfig | ExtParamConfig),
+            (GenKwConfig | SurfaceConfig | FieldConfig | EverestControl),
             Field(discriminator="type"),
         ]
     ]

ert/run_models/run_model.py

@@ -63,6 +63,7 @@ from ert.mode_definitions import MODULE_MODE
 from ert.runpaths import Runpaths
 from ert.storage import (
     Ensemble,
+    LocalStorage,
     Storage,
     open_storage,
 )
@@ -208,6 +209,10 @@ class RunModel(RunModelConfig, ABC):
     def model_post_init(self, ctx: Any) -> None:
         self._initial_realizations_mask = self.active_realizations.copy()
         self._completed_realizations_mask = [False] * len(self.active_realizations)
+
+        if LocalStorage.check_migration_needed(Path(self.storage_path)):
+            LocalStorage.perform_migration(Path(self.storage_path))
+
         self._storage = open_storage(self.storage_path, mode="w")
         self._rng = np.random.default_rng(self.random_seed)
         self._start_iteration = self.start_iteration
@@ -248,8 +253,28 @@ class RunModel(RunModelConfig, ABC):
             for key, value in self.__dict__.items()
             if key not in keys_to_drop
         }
+        settings_summary = {
+            "run_model": self.name(),
+            "num_realizations": self.runpath_config.num_realizations,
+            "num_active_realizations": self.active_realizations.count(True),
+            "num_parameters": (
+                sum(
+                    len(param_config.parameter_keys)
+                    for param_config in self.parameter_configuration
+                )
+                if hasattr(self, "parameter_configuration")
+                else "NA"
+            ),
+            "localization": getattr(
+                settings_dict.get("analysis_settings", {}), "localization", "NA"
+            ),
+        }
 
-        logger.info(f"Running '{self.name()}'\n\nSettings: {settings_dict}")
+        logger.info(
+            f"Running '{self.name()}'\n\n"
+            f"Settings summary: {settings_summary}\n\n"
+            f"Settings: {settings_dict}"
+        )
 
     @field_validator("env_vars", mode="after")
     @classmethod
@@ -813,6 +838,10 @@ class RunModel(RunModelConfig, ABC):
         if self._max_parallelism_violation.amount > 0 and isinstance(
             self._max_parallelism_violation.message, str
         ):
+            logger.info(
+                "Warning displayed to user for NUM_CPU misconfiguration:\n"
+                f"{self._max_parallelism_violation.message}"
+            )
             warnings.warn(
                 self._max_parallelism_violation.message,
                 PostSimulationWarning,

ert/services/_base_service.py

@@ -12,6 +12,7 @@ import os
 import signal
 import sys
 import threading
+import types
 from collections.abc import Callable, Mapping, Sequence
 from logging import Logger, getLogger
 from pathlib import Path
@@ -23,7 +24,7 @@ from types import FrameType
 from typing import TYPE_CHECKING, Any, Generic, Self, TypedDict, TypeVar
 
 if TYPE_CHECKING:
-    from inspect import Traceback
+    pass
 
 T = TypeVar("T", bound="BaseService")
 
@@ -87,9 +88,9 @@ class _Context(Generic[T]):
 
     def __exit__(
         self,
-        exc_type: type[BaseException],
-        exc_value: BaseException,
-        traceback: Traceback,
+        exc_type: type[BaseException] | None,
+        exc_value: BaseException | None,
+        traceback: types.TracebackType | None,
     ) -> bool:
         self._service.shutdown()
         return exc_type is None
@@ -277,7 +278,7 @@ class BaseService:
             )
 
     @classmethod
-    def start_server(cls: type[T], *args: Any, **kwargs: Any) -> _Context[T]:
+    def start_server(cls, *args: Any, **kwargs: Any) -> _Context[Self]:
         if cls._instance is not None:
             raise RuntimeError("Server already running")
         cls._instance = obj = cls(*args, **kwargs)

ert/services/ert_server.py

@@ -5,8 +5,8 @@ import logging
 import os
 import sys
 import threading
+import types
 from collections.abc import Mapping
-from inspect import Traceback
 from pathlib import Path
 from tempfile import NamedTemporaryFile
 from time import sleep
@@ -28,9 +28,9 @@ class ErtServerContext:
 
     def __exit__(
         self,
-        exc_type: type[BaseException],
-        exc_value: BaseException,
-        traceback: Traceback,
+        exc_type: type[BaseException] | None,
+        exc_value: BaseException | None,
+        traceback: types.TracebackType | None,
     ) -> bool:
         self._service.shutdown()
         return exc_type is None

ert/shared/_doc_utils/__init__.py

@@ -8,12 +8,14 @@ from docutils.statemachine import StringList
 from sphinx.util.nodes import nested_parse_with_titles
 
 
-def _parse_raw_rst(rst_string: str, node: nodes.Node, state: Any) -> None:
+def _parse_raw_rst(rst_string: str, node: nodes.Element, state: Any) -> None:
     string_list = docutils.statemachine.StringList(list(rst_string.split("\n")))
     _parse_string_list(string_list, node, state)
 
 
-def _parse_string_list(string_list: StringList, node: nodes.Node, state: Any) -> None:
+def _parse_string_list(
+    string_list: StringList, node: nodes.Element, state: Any
+) -> None:
     nested_parse_with_titles(state, string_list, node)
 
 

ert/shared/net_utils.py

@@ -1,8 +1,10 @@
+import ipaddress
 import logging
 import random
 import socket
 from functools import lru_cache
 
+import psutil
 from dns import exception, resolver, reversename
 
 
@@ -50,6 +52,7 @@ def get_machine_name() -> str:
 def find_available_socket(
     host: str | None = None,
     port_range: range = range(51820, 51840 + 1),
+    prioritize_private_ip_address: bool = False,
 ) -> socket.socket:
     """
     The default and recommended approach here is to return a bound socket to the
@@ -71,7 +74,9 @@ def find_available_socket(
 
     See e.g. implementation and comments in EvaluatorServerConfig
     """
-    current_host = host if host is not None else get_ip_address()
+    current_host = (
+        host if host is not None else get_ip_address(prioritize_private_ip_address)
+    )
 
     if port_range.start == port_range.stop:
         ports = list(range(port_range.start, port_range.stop + 1))
@@ -135,20 +140,40 @@ def get_family(host: str) -> socket.AddressFamily:
         return socket.AF_INET6
 
 
-# See https://stackoverflow.com/a/28950776
-def get_ip_address() -> str:
-    try:
-        s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
-        try:
-            s.settimeout(0)
-            # try pinging a reserved, internal address in order
-            # to determine IP representing the default route
-            s.connect(("10.255.255.255", 1))
-            address = s.getsockname()[0]
-        finally:
-            s.close()
-    except BaseException:
-        logger.warning("Cannot determine ip-address. Falling back to localhost.")
-        address = "127.0.0.1"
-    logger.debug(f"ip-address: {address}")
-    return address
+def get_ip_address(prioritize_private: bool = False) -> str:
+    """
+    Get the first (private or public) IPv4 address of the current machine on the LAN.
+    Default behaviour returns the first public IP if found, then private, then loopback.
+
+    Parameters:
+        prioritize_private (bool): If True, private IP addresses are prioritized
+
+    Returns:
+        str: The selected IP address as a string.
+    """
+    loopback = ""
+    public = ""
+    private = ""
+    interfaces = psutil.net_if_addrs()
+    for addresses in interfaces.values():
+        for address in addresses:
+            if address.family.name == "AF_INET":
+                ip = address.address
+                if ipaddress.ip_address(ip).is_loopback and not loopback:
+                    loopback = ip
+                elif ipaddress.ip_address(ip).is_private and not private:
+                    private = ip
+                elif not public:
+                    public = ip
+
+    # Select first non-empty value, based on prioritization
+    if prioritize_private:
+        selected_ip = private or public or loopback
+    else:
+        selected_ip = public or private or loopback
+
+    if selected_ip:
+        return selected_ip
+    else:
+        logger.warning("Cannot determine ip-address. Falling back to 127.0.0.1")
+        return "127.0.0.1"

ert/shared/version.py

@@ -28,7 +28,7 @@ version_tuple: VERSION_TUPLE
 commit_id: COMMIT_ID
 __commit_id__: COMMIT_ID
 
-__version__ = version = '18.0.9'
-__version_tuple__ = version_tuple = (18, 0, 9)
+__version__ = version = '19.0.0rc0'
+__version_tuple__ = version_tuple = (19, 0, 0, 'rc0')
 
-__commit_id__ = commit_id = 'ga202455d4'
+__commit_id__ = commit_id = 'gacbd663a0'

ert/storage/__init__.py

@@ -39,6 +39,8 @@ class ErtStoragePermissionError(ErtStorageException):
 def open_storage(
     path: str | os.PathLike[str], mode: ModeLiteral | Mode = "r"
 ) -> Storage:
+    _ = LocalStorage.check_migration_needed(Path(path))
+
     try:
         return LocalStorage(Path(path), Mode(mode))
     except PermissionError as err:

ert/storage/local_ensemble.py

@@ -562,12 +562,18 @@ class LocalEnsemble(BaseMode):
         df = pl.scan_parquet(group_path)
         return df
 
-    def _load_scalar_keys(
+    def load_scalar_keys(
         self,
-        keys: list[str],
+        keys: list[str] | None = None,
         realizations: int | npt.NDArray[np.int_] | None = None,
         transformed: bool = False,
     ) -> pl.DataFrame:
+        if keys is None:
+            keys = self.experiment.parameter_keys
+        elif set(keys) - set(self.experiment.parameter_keys):
+            missing = set(keys) - set(self.experiment.parameter_keys)
+            raise KeyError(f"Parameters not registered to the experiment: {missing}")
+
         df_lazy = self._load_parameters_lazy(SCALAR_FILENAME)
         df_lazy = df_lazy.select(["realization", *keys])
         if realizations is not None:
@@ -620,7 +626,7 @@ class LocalEnsemble(BaseMode):
         cardinality = next(cfg.cardinality for cfg in cfgs)
 
         if cardinality == ParameterCardinality.multiple_configs_per_ensemble_dataset:
-            return self._load_scalar_keys(
+            return self.load_scalar_keys(
                 [cfg.name for cfg in cfgs], realizations, transformed
             )
         return self._load_dataset(
@@ -647,7 +653,7 @@ class LocalEnsemble(BaseMode):
         ]
         if keys:
             return (
-                self._load_scalar_keys(keys, realizations)
+                self.load_scalar_keys(keys, realizations)
                 .drop("realization")
                 .to_numpy()
                 .T.copy()
@@ -947,7 +953,7 @@ class LocalEnsemble(BaseMode):
         data = self.load_parameters(parameter_group)
         if isinstance(data, pl.DataFrame):
             return data.drop("realization").std().to_numpy().reshape(-1)
-        return data.std("realizations")["values"].values
+        return data.std("realizations")["values"].to_numpy()
 
     def get_parameter_state(
         self, realization: int
@@ -1048,7 +1054,7 @@ class LocalEnsemble(BaseMode):
                                 pl.col(col).is_in(observed_values.implode())
                             )
 
-                    pivoted = responses.collect(engine="streaming").pivot(
+                    pivoted = responses.collect(engine="streaming").pivot(  # noqa: PD010
                         on="realization",
                         index=["response_key", *response_cls.primary_key],
                         values="values",
@@ -1224,7 +1230,7 @@ class LocalEnsemble(BaseMode):
             how="horizontal",
         )
 
-        responses_wide = responses["realization", "response_key", "values"].pivot(
+        responses_wide = responses["realization", "response_key", "values"].pivot(  # noqa: PD010
             on="response_key", values="values"
         )
 

ert/storage/local_experiment.py

@@ -16,7 +16,7 @@ from pydantic import BaseModel, Field, TypeAdapter
 from surfio import IrapSurface
 
 from ert.config import (
-    ExtParamConfig,
+    EverestControl,
     GenKwConfig,
     KnownResponseTypes,
     ParameterConfig,
@@ -66,7 +66,7 @@ _responses_adapter = TypeAdapter(  # type: ignore
 
 _parameters_adapter = TypeAdapter(  # type: ignore
     Annotated[
-        (GenKwConfig | SurfaceConfig | FieldConfig | ExtParamConfig),
+        (GenKwConfig | SurfaceConfig | FieldConfig | EverestControl),
         Field(discriminator="type"),
     ]
 )

ert/storage/local_storage.py

@@ -5,14 +5,14 @@ import logging
 import os
 import re
 import shutil
+import types
 from collections.abc import Generator, MutableSequence
 from datetime import datetime
 from functools import cached_property
 from pathlib import Path
 from tempfile import NamedTemporaryFile
 from textwrap import dedent
-from types import TracebackType
-from typing import Any
+from typing import Any, Self
 from uuid import UUID, uuid4
 
 import polars as pl
@@ -20,6 +20,7 @@ import xarray as xr
 from filelock import FileLock, Timeout
 from pydantic import BaseModel, Field
 
+import ert.storage
 from ert.config import ErtConfig, ParameterConfig, ResponseConfig
 from ert.shared import __version__
 
@@ -30,7 +31,7 @@ from .realization_storage_state import RealizationStorageState
 
 logger = logging.getLogger(__name__)
 
-_LOCAL_STORAGE_VERSION = 18
+_LOCAL_STORAGE_VERSION = 21
 
 
 class _Migrations(BaseModel):
@@ -64,6 +65,7 @@ class LocalStorage(BaseMode):
         self,
         path: str | os.PathLike[str],
         mode: Mode,
+        stage_for_migration: bool = False,
     ) -> None:
         """
         Initializes the LocalStorage instance.
@@ -74,6 +76,8 @@ class LocalStorage(BaseMode):
             The file system path to the storage.
         mode : Mode
             The access mode for the storage (read/write).
+        stage_for_migration : bool
+            Whether to avoid reloading storage to allow migration
         """
 
         self.path = Path(path).absolute()
@@ -82,9 +86,9 @@ class LocalStorage(BaseMode):
         if mode.can_write:
             self._acquire_lock()
 
-        self._experiments: dict[UUID, LocalExperiment]
-        self._ensembles: dict[UUID, LocalEnsemble]
-        self._index: _Index
+        self._experiments: dict[UUID, LocalExperiment] = {}
+        self._ensembles: dict[UUID, LocalEnsemble] = {}
+        self._index: _Index = _Index()
 
         try:
             self.version = _storage_version(self.path)
@@ -101,19 +105,18 @@ class LocalStorage(BaseMode):
                 raise ValueError(f"No index.json, but found: {errors}") from err
             self.version = _LOCAL_STORAGE_VERSION
 
-        if self.check_migration_needed(Path(self.path)):
-            if not self.can_write:
-                raise RuntimeError(
-                    f"Cannot open storage '{self.path}' in read-only mode: "
-                    f"Storage version {self.version} is too old. "
-                    f"Run ert to initiate migration."
-                )
-            else:
-                self._migrate(self.version)
+        if self.check_migration_needed(Path(self.path)) and not self.can_write:
+            raise RuntimeError(
+                f"Cannot open storage '{self.path}' in read-only mode: "
+                f"Storage version {self.version} is too old. "
+                f"Run ert to initiate migration."
+            )
 
-        self.refresh()
-        if mode.can_write:
-            self._save_index()
+        if not stage_for_migration:
+            self.reload()
+
+            if mode.can_write:
+                self._save_index()
 
     @staticmethod
     def check_migration_needed(storage_dir: Path) -> bool:
@@ -123,7 +126,7 @@ class LocalStorage(BaseMode):
             version = _LOCAL_STORAGE_VERSION
 
         if version > _LOCAL_STORAGE_VERSION:
-            raise RuntimeError(
+            raise ert.storage.ErtStorageException(
                 f"Cannot open storage '{storage_dir.absolute()}': Storage version "
                 f"{version} is newer than the current version {_LOCAL_STORAGE_VERSION}"
                 f", upgrade ert to continue, or run with a different ENSPATH"
@@ -131,7 +134,14 @@ class LocalStorage(BaseMode):
 
         return version < _LOCAL_STORAGE_VERSION
 
-    def refresh(self) -> None:
+    @staticmethod
+    def perform_migration(path: Path) -> None:
+        if LocalStorage.check_migration_needed(path):
+            with LocalStorage(path, Mode("w"), True) as storage:
+                storage._migrate(storage.version)
+                storage.reload()
+
+    def reload(self) -> None:
         """
         Reloads the index, experiments, and ensembles from the storage.
 
@@ -267,14 +277,14 @@ class LocalStorage(BaseMode):
     def _swap_path(self) -> Path:
         return self.path / self.SWAP_PATH
 
-    def __enter__(self) -> LocalStorage:
+    def __enter__(self) -> Self:
         return self
 
     def __exit__(
         self,
-        exception: Exception,
-        exception_type: type[Exception],
-        traceback: TracebackType,
+        exception: type[BaseException] | None,
+        exception_type: BaseException | None,
+        traceback: types.TracebackType | None,
     ) -> None:
         self.close()
 
@@ -504,6 +514,9 @@ class LocalStorage(BaseMode):
             to16,
             to17,
             to18,
+            to19,
+            to20,
+            to21,
         )
 
         try:
@@ -533,7 +546,7 @@ class LocalStorage(BaseMode):
 
                 self._index = self._load_index()
 
-                logger.info("storage backed up for version less than 6")
+                logger.info("Storage backed up for version less than 5")
                 print(self._legacy_storage_migration_message(bkup_path, "14.6.*"))
                 return None
             elif version < _LOCAL_STORAGE_VERSION:
@@ -551,6 +564,9 @@ class LocalStorage(BaseMode):
                     15: to16,
                     16: to17,
                     17: to18,
+                    18: to19,
+                    19: to20,
+                    20: to21,
                 }
                 for from_version in range(version, _LOCAL_STORAGE_VERSION):
                     migrations[from_version].migrate(self.path)

ert/storage/migration/to11.py

@@ -44,7 +44,7 @@ def migrate(path: Path) -> None:
                         array = ds.isel(realizations=0, drop=True)["values"]
                         realization = int(real_dir.name.split("-")[1])
 
-                        def parse_value(value: float | int | str) -> float | int | str:
+                        def parse_value(value: float | str) -> float | int | str:
                             if isinstance(value, float | int):
                                 return value
                             try:

ert/storage/migration/to18.py

@@ -4,7 +4,6 @@ info = (
     "Added localization attributes to summary observations."
     "Added RFT observations."
     "No change to current storage, only additions. "
-    "Bumping to 17 to indicate appended schema in storage."
 )
 
 

ert/storage/migration/to19.py

@@ -0,0 +1,34 @@
+import json
+from pathlib import Path
+from typing import Any
+
+info = "Add dimensionality attribute to parameters"
+
+
+def migrate_param(parameters_json: dict[str, Any]) -> dict[str, Any]:
+    new_configs = {}
+    for param_config in parameters_json.values():
+        if param_config["type"] == "surface":
+            param_config["dimensionality"] = 2
+        elif param_config["type"] == "field":
+            param_config["dimensionality"] = 3
+        else:
+            param_config["dimensionality"] = 1
+
+        new_configs[param_config["name"]] = param_config
+    return new_configs
+
+
+def migrate_parameters_for_experiment(experiment: Path) -> None:
+    with open(experiment / "parameter.json", encoding="utf-8") as fin:
+        parameters_json = json.load(fin)
+
+    new_parameter_configs = migrate_param(parameters_json)
+    Path(experiment / "parameter.json").write_text(
+        json.dumps(new_parameter_configs, indent=2), encoding="utf-8"
+    )
+
+
+def migrate(path: Path) -> None:
+    for experiment in path.glob("experiments/*"):
+        migrate_parameters_for_experiment(experiment)

ert/storage/migration/to20.py

@@ -0,0 +1,23 @@
+import json
+from pathlib import Path
+from typing import Any
+
+info = "Remove redundant .name attribute from responses."
+
+
+def config_without_name_attr(config: dict[str, Any]) -> dict[str, Any]:
+    new_json = {**config}
+    new_json.pop("name", None)
+
+    return new_json
+
+
+def migrate(path: Path) -> None:
+    for response_json_path in path.glob("experiments/*/responses.json"):
+        old_json = json.loads((response_json_path).read_text(encoding="utf-8"))
+        new_json = {
+            response_type: config_without_name_attr(config)
+            for response_type, config in old_json.items()
+        }
+
+        response_json_path.write_text(json.dumps(new_json, indent=2), encoding="utf-8")

ert/storage/migration/to21.py

@@ -0,0 +1,25 @@
+import json
+from pathlib import Path
+from typing import Any
+
+info = "Remove refcase from summary response configs"
+
+
+def config_without_refcase(summary_config: dict[str, Any]) -> dict[str, Any]:
+    new_json = {**summary_config}
+    new_json.pop("refcase", None)
+
+    return new_json
+
+
+def migrate(path: Path) -> None:
+    for response_json_path in path.glob("experiments/*/responses.json"):
+        old_json = json.loads((response_json_path).read_text(encoding="utf-8"))
+        new_json = {
+            response_type: config_without_refcase(response_config)
+            if response_config["type"] == "summary"
+            else response_config
+            for response_type, response_config in old_json.items()
+        }
+
+        response_json_path.write_text(json.dumps(new_json, indent=2), encoding="utf-8")

ert/workflow_runner.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import logging
+import types
 from concurrent import futures
 from concurrent.futures import Future
 from typing import Any, Self
@@ -127,7 +128,7 @@ class WorkflowRunner:
         self,
         exc_type: type[BaseException] | None,
         exc_value: BaseException | None,
-        traceback: Any,
+        traceback: types.TracebackType | None,
     ) -> None:
         self.wait()
 

{ert-18.0.9.dist-info → ert-19.0.0rc0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ert
-Version: 18.0.9
+Version: 19.0.0rc0
 Summary: Ensemble based Reservoir Tool (ERT)
 Author-email: Equinor ASA <fg_sib-scout@equinor.com>
 License-Expression: GPL-3.0-only