ert 16.0.9__py3-none-any.whl → 19.0.0rc2__py3-none-any.whl

everest/config/server_config.py

@@ -1,6 +1,4 @@
-import json
 import os
-from pathlib import Path
 from textwrap import dedent
 from typing import Any
 
@@ -12,15 +10,9 @@ from ert.config.queue_config import (
     SlurmQueueOptions,
     TorqueQueueOptions,
 )
-from ert.dark_storage.client import ConnInfo
+from ert.dark_storage.client import ErtClientConnectionInfo
 
-from ..strings import (
-    CERTIFICATE_DIR,
-    DETACHED_NODE_DIR,
-    HOSTFILE_NAME,
-    SERVER_STATUS,
-    SESSION_DIR,
-)
+from ..strings import SESSION_DIR
 from .simulator_config import check_removed_config
 
 
@@ -60,21 +52,11 @@ class ServerConfig(BaseModel):
             check_removed_config(data.get("queue_system"))
         return data
 
-    @staticmethod
-    def get_server_url(output_dir: str) -> str:
-        """Return the url of the server.
-
-        If server_info are given, the url is generated using that info. Otherwise
-        server information are retrieved from the hostfile
-        """
-        server_info = ServerConfig.get_server_info(output_dir)
-        return f"https://{server_info['host']}:{server_info['port']}/experiment_server"
-
     @staticmethod
     def get_server_context_from_conn_info(
-        conn_info: ConnInfo,
+        conn_info: ErtClientConnectionInfo,
     ) -> tuple[str, str, tuple[str, str]]:
-        """Get server connection context information from a storage_service ConnInfo.
+        """Get server connection context information from a ErtClientConnectionInfo.
 
         Returns a tuple containing the server URL, certificate file path,
         and authentication credentials.
@@ -84,7 +66,7 @@ class ServerConfig(BaseModel):
         This should be refactored to use the ERT Storage Client class directly instead.
 
         Args:
-            conn_info: An instance of the storage_service client ConnInfo
+            conn_info: An instance of the ErtClientConnectionInfo
 
         Returns:
             tuple: A tuple containing:
@@ -103,39 +85,8 @@ class ServerConfig(BaseModel):
 
         return url, cert_file, auth
 
-    @staticmethod
-    def get_server_info(output_dir: str) -> dict[str, Any]:
-        """Load server information from the hostfile"""
-        host_file_path = Path(ServerConfig.get_hostfile_path(output_dir))
-        if not host_file_path.exists():
-            return {"host": None, "port": None, "cert": None, "auth": None}
-
-        data = json.loads(host_file_path.read_text(encoding="utf-8"))
-
-        if not all(k in data for k in ("host", "port", "cert", "auth")):
-            raise RuntimeError("Malformed hostfile")
-        return data
-
-    @staticmethod
-    def get_detached_node_dir(output_dir: str) -> str:
-        return os.path.join(os.path.abspath(output_dir), DETACHED_NODE_DIR)
-
-    @staticmethod
-    def get_hostfile_path(output_dir: str) -> str:
-        return os.path.join(ServerConfig.get_session_dir(output_dir), HOSTFILE_NAME)
-
     @staticmethod
     def get_session_dir(output_dir: str) -> str:
         """Return path to the session directory containing information about the
         certificates and host information"""
-        return os.path.join(ServerConfig.get_detached_node_dir(output_dir), SESSION_DIR)
-
-    @staticmethod
-    def get_everserver_status_path(output_dir: str) -> str:
-        """Returns path to the everest server status file"""
-        return os.path.join(ServerConfig.get_session_dir(output_dir), SERVER_STATUS)
-
-    @staticmethod
-    def get_certificate_dir(output_dir: str) -> str:
-        """Return the path to certificate folder"""
-        return os.path.join(ServerConfig.get_session_dir(output_dir), CERTIFICATE_DIR)
+        return os.path.join(os.path.abspath(output_dir), SESSION_DIR)

everest/config/simulator_config.py

@@ -75,9 +75,22 @@ class SimulatorConfig(BaseModelWithContextSupport, extra="forbid"):
         default=None,
         description=dedent(
             """
-            Maximum allowed memory usage of a forward model.
+            Amount of memory to set aside for a forward model.
 
-            When set, a job is only allowed to use `max_memory` of memory.
+            This information is propagated to the queue system as the amount of memory
+            to reserve/book for a realization to complete. It is up to the configuration
+            of the queuing system how to treat this information, but usually it will
+            stop more realizations being assigned to a compute node if the compute nodes
+            memory is already fully booked.
+
+            Setting this number lower than the peak memory consumption of each
+            realization puts the realization at risk of being killed in an out-of-memory
+            situation. Setting this number higher than needed will give longer wait
+            times in the queue.
+
+            For the local queue system, this keyword has no effect. In that scenario,
+            you can use `max_running`  to choke the memory consumption.
+            scheduling of compute jobs.
 
             `max_memory` may be an integer value, indicating the number of
             bytes, or a string consisting of a number followed by a unit. The
@@ -94,9 +107,6 @@ class SimulatorConfig(BaseModelWithContextSupport, extra="forbid"):
             Spaces between the number and the unit are ignored, and so are any
             characters after the first. For example: 2g, 2G, and 2 GB all
             resolve to the same value: 2 gigabytes, equaling 2 * 1024**3 bytes.
-
-            If not set, or a set to zero, the allowed amount of memory is
-            unlimited.
             """
         ),
     )
@@ -132,17 +142,40 @@ class SimulatorConfig(BaseModelWithContextSupport, extra="forbid"):
         ),
     )
 
-    @field_validator("queue_system", mode="before")
+    @model_validator(mode="before")
     @classmethod
-    def default_local_queue(cls, v: Any, info: ValidationInfo) -> Any:
-        if v is None:
+    def apply_site_or_default_queue_if_no_user_queue(
+        cls, data: dict[str, Any], info: ValidationInfo
+    ) -> Any:
+        queue_system = data.get("queue_system")
+        if queue_system is None:
             options = None
             if info.context:
                 options = info.context.queue_options
-            return options or LocalQueueOptions(max_running=8)
-        return v
 
-    @field_validator("max_memory")
+            defaulted_queue_options = (
+                options.model_dump()
+                if options is not None
+                else LocalQueueOptions(max_running=8).model_dump()
+            )
+
+            user_configured_max_memory = data.get("max_memory")
+            if user_configured_max_memory is not None:
+                cls.validate_max_memory(max_memory=user_configured_max_memory)
+                defaulted_queue_options["realization_memory"] = (
+                    user_configured_max_memory
+                )
+
+            user_configured_cores_per_node = data.get("cores_per_node")
+            if user_configured_cores_per_node is not None:
+                defaulted_queue_options["num_cpu"] = user_configured_cores_per_node
+
+            data["queue_system"] = defaulted_queue_options
+            data["max_memory"] = None
+
+        return data
+
+    @field_validator("max_memory", mode="before")
     @classmethod
     def validate_max_memory(cls, max_memory: int | str | None) -> str | None:
         if max_memory is None:
@@ -163,14 +196,26 @@ class SimulatorConfig(BaseModelWithContextSupport, extra="forbid"):
 
     @model_validator(mode="after")
     def update_max_memory(config: "SimulatorConfig") -> "SimulatorConfig":
+        if config.max_memory is None:
+            return config
+        parsed_max_memory = (
+            parse_string_to_bytes(config.max_memory)
+            if type(config.max_memory) is str
+            else int(config.max_memory)
+        )
         if (
-            config.max_memory is not None
-            and config.queue_system is not None
+            config.queue_system is not None
             and config.queue_system.realization_memory == 0
         ):
-            config.queue_system.realization_memory = (
-                parse_string_to_bytes(config.max_memory)
-                if type(config.max_memory) is str
-                else int(config.max_memory)
+            config.queue_system.realization_memory = parsed_max_memory
+        elif (
+            config.queue_system is not None
+            and config.queue_system.realization_memory > 0
+            and config.queue_system.realization_memory != parsed_max_memory
+        ):
+            raise ConfigValidationError(
+                "Ambiguous configuration of realization_memory. "
+                "Specify either max_memory or realization_memory, not both"
             )
+
         return config

everest/config/utils.py

@@ -1,108 +1,28 @@
-from copy import deepcopy
-from typing import Any, Literal
+from ert.config import EverestControl, SamplerConfig
 
-from .control_config import ControlConfig
-from .control_variable_config import ControlVariableGuessListConfig
-from .sampler_config import SamplerConfig
 
-
-class FlattenedControls:
-    def __init__(self, controls: list[ControlConfig]) -> None:
-        control_dicts = _get_control_dicts(controls)
-        self.names = [control["name"] for control in control_dicts]
-        self.types: list[Literal["real", "integer"]] = [
-            control["control_type"] for control in control_dicts
-        ]
-        self.initial_guesses = [control["initial_guess"] for control in control_dicts]
-        self.lower_bounds = [control["min"] for control in control_dicts]
-        self.upper_bounds = [control["max"] for control in control_dicts]
-        self.scaled_ranges = [control["scaled_range"] for control in control_dicts]
-        self.enabled = [control["enabled"] for control in control_dicts]
-        self.perturbation_magnitudes = [
-            control["perturbation_magnitude"] for control in control_dicts
-        ]
-        self.perturbation_types = [
-            control["perturbation_type"] for control in control_dicts
-        ]
-        self.samplers, self.sampler_indices = _get_samplers(controls)
-
-
-def _get_control_dicts(controls: list[ControlConfig]) -> list[dict[str, Any]]:
-    def _inject_defaults(control: ControlConfig, var_dict: dict[str, Any]) -> None:
-        for key in [
-            "type",
-            "initial_guess",
-            "control_type",
-            "enabled",
-            "min",
-            "max",
-            "perturbation_type",
-            "perturbation_magnitude",
-            "scaled_range",
-        ]:
-            if var_dict.get(key) is None:
-                var_dict[key] = getattr(control, key)
-
-    control_dicts: list[dict[str, Any]] = []
-    for control in controls:
-        for variable in control.variables:
-            if isinstance(variable, ControlVariableGuessListConfig):
-                for index, guess in enumerate(variable.initial_guess, start=1):
-                    var_dict = deepcopy(variable.model_dump())
-                    var_dict["name"] = (control.name, variable.name, index)
-                    var_dict["initial_guess"] = guess
-                    _inject_defaults(control, var_dict)
-                    control_dicts.append(var_dict)
-            else:
-                var_dict = deepcopy(variable.model_dump())
-                var_dict["name"] = (
-                    (control.name, variable.name)
-                    if variable.index is None
-                    else (control.name, variable.name, variable.index)
-                )
-                _inject_defaults(control, var_dict)
-                control_dicts.append(var_dict)
-    return control_dicts
-
-
-def _get_samplers(
-    controls: list[ControlConfig],
+def get_samplers(
+    controls: list[EverestControl],
 ) -> tuple[list[SamplerConfig | None], list[int]]:
-    samplers: list[SamplerConfig | None] = []
-    sampler_indices: list[int] = []
-
-    default_sampler_index: int | None = None
-
-    for control in controls:
-        control_sampler_index: int | None = None
-
-        for variable in control.variables:
-            if variable.sampler is not None:
-                # Use the sampler of the variable:
-                samplers.append(variable.sampler)
-                variable_sampler_index = len(samplers) - 1
-            elif control.sampler is not None:
-                # Use the sampler of the control:
-                if control_sampler_index is None:
-                    samplers.append(control.sampler)
-                    control_sampler_index = len(samplers) - 1
-                variable_sampler_index = control_sampler_index
-            else:
-                # Use the default sampler:
-                if default_sampler_index is None:
-                    samplers.append(None)
-                    default_sampler_index = len(samplers) - 1
-                variable_sampler_index = default_sampler_index
-
-            if isinstance(variable, ControlVariableGuessListConfig):
-                sampler_indices.extend(
-                    [variable_sampler_index] * len(variable.initial_guess)
-                )
-            else:
-                sampler_indices.append(variable_sampler_index)
-
-    return samplers, sampler_indices
-
-
-def flatten_controls(controls: list[ControlConfig]) -> FlattenedControls:
-    return FlattenedControls(controls)
+    """
+    Create a list of unique samplers, and a list mapping variable index
+    to sampler index. I.e., this points each control variable to
+    a sampler by index.
+    """
+    flattened_samplers = [
+        sampler for control in controls for sampler in control.samplers
+    ]
+    unique_samplers: list[SamplerConfig | None] = []
+    variable_to_unique_sampler_index: list[int] = []
+    for sampler in flattened_samplers:
+        try:
+            unique_sampler_index = next(
+                i for i, s in enumerate(unique_samplers) if s == sampler
+            )
+        except StopIteration:
+            unique_sampler_index = len(unique_samplers)
+            unique_samplers.append(sampler)
+
+        variable_to_unique_sampler_index.append(unique_sampler_index)
+
+    return unique_samplers, variable_to_unique_sampler_index

everest/config/validation_utils.py

@@ -1,6 +1,7 @@
 import errno
 import os
 import tempfile
+import types
 from collections import Counter
 from collections.abc import Sequence
 from pathlib import Path
@@ -15,7 +16,7 @@ from everest.util.forward_models import (
     parse_forward_model_file,
 )
 
-from .install_job_config import InstallJobConfig
+from .install_job_config import InstallForwardModelStepConfig
 
 _VARIABLE_ERROR_MESSAGE = (
     "Variable {name} must define {variable_type} value either"
@@ -39,6 +40,8 @@ _RESERVED_WORDS = [
     "total_objective_value",
 ]
 
+_OVERWRITE_MESSAGE = "Are you overwriting other parts of install_data?"
+
 
 class InstallDataContext:
     def __init__(
@@ -70,28 +73,44 @@ class InstallDataContext:
                 source = item.source
                 target = item.target
 
-            if "<GEO_ID>" not in source:
+            if "<REALIZATION_ID>" not in source:
                 self._set_symlink(source, target, None)
 
         return self
 
     def _set_symlink(self, source: str, target: str, realization: int | None) -> None:
         if realization is not None:
-            source = source.replace("<GEO_ID>", str(realization))
-            target = target.replace("<GEO_ID>", str(realization))
+            source = source.replace("<REALIZATION_ID>", str(realization))
+            target = target.replace("<REALIZATION_ID>", str(realization))
 
         tmp_target = Path(self._temp_dir.name) / Path(target)
         if tmp_target.exists():
+            if tmp_target.is_dir() and not tmp_target.is_symlink():
+                raise ValueError(
+                    "Cannot make symlink due to existing directory at target location"
+                    f" {tmp_target}. " + _OVERWRITE_MESSAGE
+                )
             tmp_target.unlink()
-        tmp_target.parent.mkdir(parents=True, exist_ok=True)
-        tmp_target.symlink_to(as_abs_path(source, self._config_dir))
+        try:
+            tmp_target.parent.mkdir(parents=True, exist_ok=True)
+            tmp_target.symlink_to(as_abs_path(source, self._config_dir))
+        except FileExistsError as err:
+            raise ValueError(
+                f"Cannot install data {source} into {target} due to existing "
+                "file or directory. " + _OVERWRITE_MESSAGE
+            ) from err
 
     def add_links_for_realization(self, realization: int) -> None:
         for data in self._install_data:
-            if data.source is not None and "<GEO_ID>" in data.source:
+            if data.source is not None and "<REALIZATION_ID>" in data.source:
                 self._set_symlink(data.source, data.target, realization)
 
-    def __exit__(self, exc_type: Any, exc_value: Any, exc_tb: Any) -> None:
+    def __exit__(
+        self,
+        exc_type: type[BaseException] | None,
+        exc_value: BaseException | None,
+        exc_tb: types.TracebackType | None,
+    ) -> None:
         if self._temp_dir:
             self._temp_dir.cleanup()
         os.chdir(self._cwd)
@@ -177,7 +196,7 @@ def unique_items(items: Sequence[T]) -> Sequence[T]:
 
 
 def valid_range(range_value: tuple[float, float]) -> tuple[float, float]:
-    if range_value[0] >= range_value[1]:
+    if range_value is not None and range_value[0] >= range_value[1]:
         raise ValueError("scaled_range must be a valid range [a, b], where a < b.")
     return range_value
 
@@ -229,8 +248,8 @@ def as_abs_path(path: str, config_dir: str) -> str:
 
 
 def expand_model_id_paths(path_source: str, realizations: list[int]) -> list[str]:
-    if "<GEO_ID>" in path_source:
-        return [path_source.replace("<GEO_ID>", str(r)) for r in realizations]
+    if "<REALIZATION_ID>" in path_source:
+        return [path_source.replace("<REALIZATION_ID>", str(r)) for r in realizations]
     return [path_source]
 
 
@@ -238,7 +257,8 @@ def check_path_exists(
     path_source: str, config_path: Path | None, realizations: list[int]
 ) -> None:
     """Check if the given path exists. If the given path contains <CONFIG_PATH>
-    or GEO_ID they will be expanded and all instances of expanded paths need to exist.
+    or REALIZATION_ID they will be expanded and all instances of expanded paths
+    need to exist.
     """
     if not isinstance(path_source, str):
         raise ValueError(
@@ -272,8 +292,7 @@ def check_writeable_path(path_source: str, config_path: Path) -> None:
             if os.access(path, os.W_OK | os.X_OK):
                 break
         elif os.path.isfile(path):
-            # path is a file, cannot create folder
-            raise ValueError(f"File {path} exists, cannot create path {path_source}")
+            raise ValueError(f"{path} is a file, cannot create folders inside it")
         parent = os.path.dirname(path)
         if parent == path:  # ie, if path is root
             break
@@ -284,7 +303,7 @@ def check_writeable_path(path_source: str, config_path: Path) -> None:
 
 
 def validate_forward_model_configs(
-    forward_model: list[str], install_jobs: list[InstallJobConfig]
+    forward_model: list[str], install_jobs: list[InstallForwardModelStepConfig]
 ) -> None:
     if not forward_model:
         return

everest/config_file_loader.py

@@ -1,9 +1,8 @@
-#!/usr/bin/env python
-
 import logging
 import os
 import re
 from io import StringIO
+from pathlib import Path
 from typing import Any
 
 import jinja2
@@ -25,29 +24,28 @@ SUBSTITUTION_PATTERN = r"(r\{\{.*?\}\})"
 
 # Jinja vars which should NOT be included in definitions portion of config.
 ERT_CONFIG_TEMPLATES = {
-    "realization": "GEO_ID",
+    "realization": "REALIZATION_ID",
     "runpath_file": "RUNPATH_FILE",
 }
 
 
 def load_yaml(file_name: str, safe: bool = False) -> dict[str, Any]:
-    with open(file_name, encoding="utf-8") as input_file:
-        input_data: list[str] = input_file.readlines()
-        try:
-            yaml = YAML(typ="safe", pure=True) if safe else YAML()
-            yaml.preserve_quotes = True
-            return yaml.load("".join(input_data))
-        except YAMLError as exc:
-            if hasattr(exc, "problem_mark"):
-                mark = exc.problem_mark
-                raise YAMLError(
-                    str(exc)
-                    + "\nError in line: {}\n {}^)".format(
-                        input_data[mark.line], " " * mark.column
-                    )
-                ) from exc
-            else:
-                raise YAMLError(str(exc)) from exc
+    input_data = Path(file_name).read_text(encoding="utf-8").splitlines()
+    try:
+        yaml = YAML(typ="safe", pure=True) if safe else YAML()
+        yaml.preserve_quotes = True
+        return yaml.load("\n".join(input_data))
+    except YAMLError as exc:
+        if hasattr(exc, "problem_mark"):
+            mark = exc.problem_mark
+            raise YAMLError(
+                str(exc)
+                + "\nError in line: {}\n {}^)".format(
+                    input_data[mark.line], " " * mark.column
+                )
+            ) from exc
+        else:
+            raise YAMLError(str(exc)) from exc
 
 
 def _get_definitions(
@@ -67,7 +65,7 @@ def _get_definitions(
                     f"Internal key {key} specified by user as {defs[key]}. "
                     f"Overriding as {val}"
                 )
-            defs[key] = f"<{val}>"  # ert uses <GEO_ID> as format
+            defs[key] = f"<{val}>"  # ert uses <REALIZATION_ID> as format
     else:
         logging.getLogger(EVEREST).warning("Empty configuration file provided!")
 
@@ -160,6 +158,17 @@ def yaml_file_to_substituted_config_dict(config_path: str) -> dict[str, Any]:
     # Replace in definitions
     config = jenv.from_string(txt).render(**definitions)
 
+    # Handle deprecated use of <GEO_ID>:
+    config, count = re.subn(
+        rf"<\s*{re.escape('GEO_ID')}\s*>", "<REALIZATION_ID>", config
+    )
+    if count > 0:
+        message = r"<GEO_ID> is deprecated, please replace with 'r{{realization}}'."
+        print(message)
+        logging.getLogger(EVEREST).warning(
+            "Deprecated key <GEO_ID> replaced by <REALIZATION_ID>."
+        )
+
     # Load the config with definitions again as yaml
     yaml = YAML(typ="safe", pure=True).load(config)
 

everest/detached/__init__.py

@@ -4,28 +4,22 @@ Client methods for interacting with everserver
 
 from .client import (
     PROXY,
-    ExperimentState,
-    everserver_status,
     server_is_running,
     start_experiment,
     start_monitor,
     start_server,
     stop_server,
-    update_everserver_status,
     wait_for_server,
     wait_for_server_to_stop,
 )
 
 __all__ = [
     "PROXY",
-    "ExperimentState",
-    "everserver_status",
     "server_is_running",
     "start_experiment",
     "start_monitor",
     "start_server",
     "stop_server",
-    "update_everserver_status",
     "wait_for_server",
     "wait_for_server_to_stop",
 ]