ert 17.1.7__py3-none-any.whl → 18.0.0__py3-none-any.whl

everest/bin/config_branch_script.py

@@ -5,8 +5,6 @@ from pathlib import Path
 from textwrap import dedent
 from typing import Any
 
-from ruamel.yaml import YAML
-
 from everest.bin.utils import setup_logging
 from everest.config import EverestConfig
 from everest.config_file_loader import load_yaml
@@ -139,10 +137,9 @@ def config_branch_entry(args: list[str] | None = None) -> None:
             conf_controls=yml_config["controls"], opt_controls=opt_controls
         )
 
-        yaml = YAML()
-        yaml.indent(mapping=2, sequence=4, offset=2)
-        yaml.preserve_quotes = True
-        yaml.dump(yml_config, Path(options.output_config))
+        EverestConfig.write_dict_to_file(
+            yml_config, Path(options.output_config), safe_and_pure=False
+        )
         print(f"New config file {options.output_config} created.")
 
 

everest/bin/everconfigdump_script.py

@@ -3,8 +3,6 @@
 import argparse
 import sys
 
-from ruamel.yaml import YAML
-
 from everest.config import EverestConfig
 
 
@@ -26,13 +24,7 @@ def _build_args_parser() -> argparse.ArgumentParser:
 def config_dump_entry(args: list[str] | None = None) -> None:
     parser = _build_args_parser()
     options = parser.parse_args(args)
-
-    config = EverestConfig.load_file(options.config_file)
-
-    yaml = YAML(typ="safe", pure=True)
-    yaml.indent = 2
-    yaml.default_flow_style = False
-    yaml.dump(config.to_dict(), sys.stdout)
+    EverestConfig.load_file(options.config_file).write_to_file(sys.stdout)
 
 
 if __name__ == "__main__":

everest/bin/everest_script.py

@@ -3,7 +3,6 @@ import argparse
 import asyncio
 import json
 import logging
-import os
 import signal
 import socket
 import threading
@@ -11,9 +10,11 @@ from functools import partial
 from pathlib import Path
 from textwrap import dedent
 
+import anyio
+
 from _ert.threading import ErtThread
 from ert.config import QueueSystem
-from ert.services import StorageService
+from ert.services import ErtServer
 from ert.storage.local_experiment import ExperimentState
 from everest.config import EverestConfig, ServerConfig
 from everest.detached import (
@@ -168,7 +169,7 @@ async def run_everest(options: argparse.Namespace) -> None:
     )
 
     try:
-        StorageService.session(
+        ErtServer.session(
             Path(ServerConfig.get_session_dir(options.config.output_dir)), timeout=1
         )
         server_running = True
@@ -193,18 +194,27 @@ async def run_everest(options: argparse.Namespace) -> None:
         job_name = fm_job.split()[0]
         logger.info(f"Everest forward model contains job {job_name}")
 
-    if os.path.exists(options.config.simulation_dir) and any(
-        os.listdir(options.config.simulation_dir)
-    ):
+    async def directory_is_nonempty(path: Path) -> bool:
+        try:
+            async for _ in anyio.Path(path).iterdir():
+                return True
+        except OSError:  # Raised if the directory is empty
+            return False
+        else:
+            return False
+
+    if await anyio.Path(
+        options.config.simulation_dir
+    ).exists() and await directory_is_nonempty(options.config.simulation_dir):
         warn_user_that_runpath_is_nonempty()
     if not options.skip_prompt:
         show_scaled_controls_warning()
 
     try:
-        output_dir = options.config.output_dir
-        config_file = options.config.config_file
-        save_config_path = os.path.join(output_dir, config_file)
-        options.config.dump(save_config_path)
+        output_dir = Path(options.config.output_dir)
+        options.config.write_to_file(
+            output_dir / options.config.config_file, drop_config_path=True
+        )
     except (OSError, LookupError) as e:
         logger.error(f"Failed to save optimization config: {e}")
 
@@ -223,7 +233,7 @@ async def run_everest(options: argparse.Namespace) -> None:
 
     print("Waiting for server ...")
     logger.debug("Waiting for response from everserver")
-    client = StorageService.session(
+    client = ErtServer.session(
         Path(ServerConfig.get_session_dir(options.config.output_dir))
     )
     wait_for_server(client, timeout=600)

everest/bin/kill_script.py

@@ -12,7 +12,7 @@ from functools import partial
 from pathlib import Path
 from typing import Any
 
-from ert.services import StorageService
+from ert.services import ErtServer
 from everest.bin.utils import setup_logging
 from everest.config import EverestConfig, ServerConfig
 from everest.detached import stop_server, wait_for_server_to_stop
@@ -74,7 +74,7 @@ def _handle_keyboard_interrupt(signal: int, _: Any, after: bool = False) -> None
 
 def kill_everest(options: argparse.Namespace) -> None:
     try:
-        client = StorageService.session(
+        client = ErtServer.session(
             Path(ServerConfig.get_session_dir(options.config.output_dir)), timeout=1
         )
         server_context = ServerConfig.get_server_context_from_conn_info(

everest/bin/monitor_script.py

@@ -7,7 +7,7 @@ from functools import partial
 from pathlib import Path
 from textwrap import dedent
 
-from ert.services import StorageService
+from ert.services import ErtServer
 from ert.storage import ErtStorageException, ExperimentState
 from everest.config import EverestConfig, ServerConfig
 from everest.everest_storage import EverestStorage
@@ -82,7 +82,7 @@ def _build_args_parser() -> argparse.ArgumentParser:
 def monitor_everest(options: argparse.Namespace) -> None:
     config: EverestConfig = options.config
     try:
-        with StorageService.session(
+        with ErtServer.session(
             Path(ServerConfig.get_session_dir(config.output_dir)), timeout=1
         ) as client:
             server_context = ServerConfig.get_server_context_from_conn_info(

everest/bin/utils.py

@@ -26,7 +26,7 @@ from ert.ensemble_evaluator import (
 from ert.ensemble_evaluator.event import EndEvent
 from ert.logging import LOGGING_CONFIG
 from ert.plugins.plugin_manager import ErtPluginManager
-from ert.services import StorageService
+from ert.services import ErtServer
 from ert.storage import (
     ExperimentStatus,
     open_storage,
@@ -39,10 +39,13 @@ from everest.detached import (
     stop_server,
     wait_for_server_to_stop,
 )
-from everest.simulator import JOB_FAILURE, JOB_RUNNING, JOB_SUCCESS
 from everest.strings import EVEREST, OPT_PROGRESS_ID, SIM_PROGRESS_ID
 from everest.util import makedirs_if_needed
 
+JOB_SUCCESS = "Finished"
+JOB_RUNNING = "Running"
+JOB_FAILURE = "Failed"
+
 
 def cleanup_logging() -> None:
     os.environ.pop("ERT_LOG_DIR", None)
@@ -103,7 +106,7 @@ def handle_keyboard_interrupt(signum: int, _: Any, options: argparse.Namespace)
             "The optimization will be stopped and the program will exit..."
         )
         try:
-            client = StorageService.session(
+            client = ErtServer.session(
                 Path(ServerConfig.get_session_dir(options.config.output_dir))
             )
             server_context = ServerConfig.get_server_context_from_conn_info(

everest/config/__init__.py

@@ -1,3 +1,5 @@
+from ert.config import SamplerConfig
+
 from .control_config import ControlConfig
 from .control_variable_config import (
     ControlVariableConfig,
@@ -6,6 +8,7 @@ from .control_variable_config import (
 from .cvar_config import CVaRConfig
 from .environment_config import EnvironmentConfig
 from .everest_config import EverestConfig, EverestValidationError
+from .forward_model_config import ForwardModelStepConfig
 from .input_constraint_config import InputConstraintConfig
 from .install_data_config import InstallDataConfig
 from .install_job_config import InstallJobConfig
@@ -14,7 +17,6 @@ from .model_config import ModelConfig
 from .objective_function_config import ObjectiveFunctionConfig
 from .optimization_config import OptimizationConfig
 from .output_constraint_config import OutputConstraintConfig
-from .sampler_config import SamplerConfig
 from .server_config import ServerConfig
 from .simulator_config import SimulatorConfig
 from .well_config import WellConfig
@@ -28,6 +30,7 @@ __all__ = [
     "EnvironmentConfig",
     "EverestConfig",
     "EverestValidationError",
+    "ForwardModelStepConfig",
     "InputConstraintConfig",
     "InstallDataConfig",
     "InstallJobConfig",

everest/config/control_config.py

@@ -12,13 +12,12 @@ from typing import (
 from pydantic import AfterValidator, BaseModel, ConfigDict, Field, model_validator
 from ropt.enums import PerturbationType, VariableType
 
-from ert.config import ExtParamConfig
+from ert.config import ExtParamConfig, SamplerConfig
 
 from .control_variable_config import (
     ControlVariableConfig,
     ControlVariableGuessListConfig,
 )
-from .sampler_config import SamplerConfig
 from .validation_utils import (
     control_variables_validation,
     no_dots_in_string,
@@ -314,8 +313,68 @@ class ControlConfig(BaseModel):
     )
 
     def to_ert_parameter_config(self) -> ExtParamConfig:
+        types = []
+        initial_guesses = []
+        control_types = []
+        enabled = []
+        mins = []
+        maxs = []
+        perturbation_types = []
+        perturbation_magnitudes = []
+        scaled_ranges = []
+        samplers = []
+
+        def _parse_variable(
+            variable: ControlVariableConfig | ControlVariableGuessListConfig,
+            initial_guess: float | None = None,
+        ) -> None:
+            types.append(self.type)
+            initial_guesses.append(
+                initial_guess
+                if initial_guess is not None
+                else variable.initial_guess
+                if variable.initial_guess is not None
+                else self.initial_guess
+            )
+            control_types.append(self.control_type)
+            enabled.append(
+                variable.enabled if variable.enabled is not None else self.enabled
+            )
+            mins.append(variable.min if variable.min is not None else self.min)
+            maxs.append(variable.max if variable.max is not None else self.max)
+            perturbation_types.append(variable.perturbation_type)
+            perturbation_magnitudes.append(
+                variable.perturbation_magnitude
+                if variable.perturbation_magnitude is not None
+                else self.perturbation_magnitude
+            )
+            scaled_ranges.append(
+                variable.scaled_range
+                if variable.scaled_range is not None
+                else self.scaled_range
+            )
+            samplers.append(variable.sampler or self.sampler)
+
+        for variable in self.variables:
+            if isinstance(variable, ControlVariableConfig):
+                _parse_variable(variable)
+            else:
+                for initial_guess in variable.initial_guess:
+                    _parse_variable(variable, initial_guess=initial_guess)
+
         return ExtParamConfig(
             name=self.name,
             input_keys=self.formatted_control_names,
+            input_keys_dotdash=self.formatted_control_names_dotdash,
             output_file=self.name + ".json",
+            types=types,
+            initial_guesses=initial_guesses,
+            control_types=control_types,
+            enabled=enabled,
+            min=mins,
+            max=maxs,
+            perturbation_types=perturbation_types,
+            perturbation_magnitudes=perturbation_magnitudes,
+            scaled_ranges=scaled_ranges,
+            samplers=samplers,
         )

everest/config/control_variable_config.py

@@ -12,7 +12,8 @@ from pydantic import (
 )
 from ropt.enums import VariableType
 
-from .sampler_config import SamplerConfig
+from ert.config import SamplerConfig
+
 from .validation_utils import no_dots_in_string, valid_range
 
 

everest/config/everest_config.py

@@ -2,7 +2,6 @@ import logging
 import os
 from argparse import ArgumentParser
 from copy import copy
-from io import StringIO
 from itertools import chain
 from pathlib import Path
 from sys import float_info
@@ -12,6 +11,7 @@ from typing import (
     Any,
     Optional,
     Self,
+    TextIO,
     TypeVar,
     no_type_check,
 )
@@ -30,7 +30,12 @@ from pydantic_core.core_schema import ValidationInfo
 from ruamel.yaml import YAML, YAMLError
 
 from ert.base_model_context import BaseModelWithContextSupport, use_runtime_plugins
-from ert.config import ConfigWarning, EverestObjectivesConfig, QueueSystem
+from ert.config import (
+    ConfigWarning,
+    EverestConstraintsConfig,
+    EverestObjectivesConfig,
+    QueueSystem,
+)
 from ert.plugins import get_site_plugins
 from everest.config.install_template_config import InstallTemplateConfig
 from everest.config.server_config import ServerConfig
@@ -1076,22 +1081,25 @@ to read summary data from forward model, do:
         except ValueError as e:
             parser.error(f"Loading config file <{config_path}> failed with: {e}")
 
-    def dump(self, fname: str | None = None) -> str | None:
-        """Write a config dict to file or return it if fname is None."""
-        stripped_conf = self.to_dict()
-
-        del stripped_conf["config_path"]
+    def write_to_file(
+        self, output_file: str | Path | TextIO, drop_config_path: bool = False
+    ) -> None:
+        dictconf = self.to_dict()
+        if drop_config_path:
+            del dictconf["config_path"]
+        if isinstance(output_file, str):
+            output_file = Path(output_file)
+        self.write_dict_to_file(dictconf, output_file)
 
-        yaml = YAML(typ="safe", pure=True)
+    @staticmethod
+    def write_dict_to_file(
+        config: dict[str, Any], output_file: Path | TextIO, safe_and_pure: bool = True
+    ) -> None:
+        yaml = YAML(typ="safe", pure=True) if safe_and_pure else YAML()
+        yaml.indent(mapping=2, sequence=4, offset=2)
+        yaml.preserve_quotes = True
         yaml.default_flow_style = False
-        if fname is None:
-            with StringIO() as sio:
-                yaml.dump(stripped_conf, sio)
-                return sio.getvalue()
-
-        yaml.dump(stripped_conf, Path(fname))
-
-        return None
+        yaml.dump(config, output_file)
 
     @property
     def server_queue_system(self) -> QueueSystem:
@@ -1108,3 +1116,17 @@ to read summary data from forward model, do:
             scales=[o.scale for o in self.objective_functions],
             objective_types=[o.type for o in self.objective_functions],
         )
+
+    def create_ert_output_constraints_config(self) -> EverestConstraintsConfig | None:
+        if len(self.output_constraints) == 0:
+            return None
+
+        constraint_names = [c.name for c in self.output_constraints]
+        return EverestConstraintsConfig(
+            keys=constraint_names,
+            input_files=constraint_names,
+            scales=[c.scale for c in self.output_constraints],
+            targets=[c.target for c in self.output_constraints],
+            upper_bounds=[c.upper_bound for c in self.output_constraints],
+            lower_bounds=[c.lower_bound for c in self.output_constraints],
+        )

everest/config/forward_model_config.py

@@ -9,7 +9,9 @@ from pydantic import (
 )
 
 from ert.base_model_context import BaseModelWithContextSupport
-from ert.config import ForwardModelStep
+from ert.config import (
+    SiteOrUserForwardModelStep,
+)
 
 
 class ForwardModelResult(BaseModelWithContextSupport):
@@ -74,8 +76,8 @@ class ForwardModelStepConfig(BaseModelWithContextSupport):
         return values
 
     def to_ert_forward_model_step(
-        self, installed_fm_steps: dict[str, ForwardModelStep]
-    ) -> ForwardModelStep:
+        self, installed_fm_steps: dict[str, SiteOrUserForwardModelStep]
+    ) -> SiteOrUserForwardModelStep:
         fm_name, *arglist = self.job.split()
         match fm_name:
             # All three reservoir simulator fm_steps map to

everest/config/install_data_config.py

@@ -9,11 +9,13 @@ from typing import Any
 
 from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
 
-from ert.config import ForwardModelStep
+from ert.config.forward_model_step import (
+    SiteOrUserForwardModelStep,
+)
 
 
 def _is_dir_all_model(source: str, model_realizations: list[int]) -> bool:
-    """Expands <GEO_ID> for all realizations and if:
+    """Expands <REALIZATION_ID> for all realizations and if:
     - all are directories, returns True,
     - all are files, returns False,
     - some are non-existing, raises an AssertionError
@@ -21,7 +23,7 @@ def _is_dir_all_model(source: str, model_realizations: list[int]) -> bool:
 
     is_dir = []
     for model_realization in model_realizations:
-        model_source = source.replace("<GEO_ID>", str(model_realization))
+        model_source = source.replace("<REALIZATION_ID>", str(model_realization))
         if not os.path.exists(model_source):
             msg = (
                 "Expected source to exist for data installation, "
@@ -126,8 +128,8 @@ class InstallDataConfig(BaseModel):
         config_directory: str,
         output_directory: str,
         model_realizations: list[int],
-        installed_fm_steps: dict[str, ForwardModelStep],
-    ) -> ForwardModelStep:
+        installed_fm_steps: dict[str, SiteOrUserForwardModelStep],
+    ) -> SiteOrUserForwardModelStep:
         target = self.target
 
         def _missing_fm_msg(fm_name: str) -> str:

everest/config/install_job_config.py

@@ -7,7 +7,7 @@ from pydantic import BaseModel, Field, model_validator
 
 from ert.config import (
     ExecutableWorkflow,
-    ForwardModelStep,
+    UserInstalledForwardModelStep,
     forward_model_step_from_config_contents,
     workflow_job_from_file,
 )
@@ -57,12 +57,16 @@ class InstallJobConfig(BaseModel, extra="forbid"):
 
 
 class InstallForwardModelStepConfig(InstallJobConfig):
-    def to_ert_forward_model_step(self, config_directory: str) -> ForwardModelStep:
+    def to_ert_forward_model_step(
+        self, config_directory: str
+    ) -> UserInstalledForwardModelStep:
         if self.executable is not None:
             executable = Path(self.executable)
             if not executable.is_absolute():
                 executable = Path(config_directory) / executable
-            return ForwardModelStep(name=self.name, executable=str(executable))
+            return UserInstalledForwardModelStep(
+                name=self.name, executable=str(executable)
+            )
         else:
             assert (
                 self.source is not None

everest/config/install_template_config.py

@@ -4,7 +4,7 @@ from textwrap import dedent
 
 from pydantic import BaseModel, Field
 
-from ert.config import ForwardModelStep
+from ert.config import SiteOrUserForwardModelStep
 from everest.strings import EVEREST
 
 
@@ -38,9 +38,9 @@ class InstallTemplateConfig(BaseModel, extra="forbid"):
     def to_ert_forward_model_step(
         self,
         control_names: list[str],
-        installed_fm_steps: dict[str, ForwardModelStep],
+        installed_fm_steps: dict[str, SiteOrUserForwardModelStep],
         well_path: str,
-    ) -> ForwardModelStep:
+    ) -> SiteOrUserForwardModelStep:
         fm_step_instance = copy.deepcopy(installed_fm_steps.get("template_render"))
         if fm_step_instance is None:
             raise KeyError(

everest/config/optimization_config.py

@@ -1,11 +1,12 @@
+import importlib
 import logging
 from textwrap import dedent
 from typing import Any, Self
 
 from pydantic import BaseModel, Field, model_validator
+from ropt.workflow import find_optimizer_plugin, validate_optimizer_options
 
 from everest.config.cvar_config import CVaRConfig
-from everest.optimizer.utils import get_ropt_plugin_manager
 from everest.strings import EVEREST
 
 
@@ -346,15 +347,27 @@ class OptimizationConfig(BaseModel, extra="forbid"):
             else f"{self.backend}/{self.algorithm}"
         )
 
-        plugin_manager = get_ropt_plugin_manager()
-        plugin_name = plugin_manager.get_plugin_name("optimizer", algorithm)
+        try:
+            plugin_name = find_optimizer_plugin(algorithm)
+        except ValueError:
+            raise
+        except Exception as exc:
+            ert_version = importlib.metadata.version("ert")
+            ropt_version = importlib.metadata.version("ropt")
+            msg = (
+                f"Error while initializing ropt:\n\n{exc}.\n\n"
+                "There may a be version mismatch between "
+                f"ERT ({ert_version}) and ropt ({ropt_version})\n"
+                "If the installation is correct, please report this as a bug."
+            )
+            raise RuntimeError(msg) from exc
         if plugin_name is None:
             raise ValueError(f"Optimizer algorithm '{algorithm}' not found")
         self._optimization_plugin_name = plugin_name
 
-        plugin_manager.get_plugin("optimizer", algorithm).validate_options(
-            self.algorithm, self.options or self.backend_options
-        )
+        options = self.options or self.backend_options
+        if options:
+            validate_optimizer_options(algorithm, options)
 
         self.backend = None
         self.algorithm = algorithm

everest/config/output_constraint_config.py

@@ -74,10 +74,16 @@ class OutputConstraintConfig(BaseModel, extra="forbid"):
     @model_validator(mode="before")
     @classmethod
     def validate_target_or_bounds(cls, values: dict[str, Any]) -> dict[str, Any]:
-        if "target" in values and ("lower_bound" in values or "upper_bound" in values):
+        if (values.get("target") is not None) and (
+            "lower_bound" in values or "upper_bound" in values
+        ):
             raise ValueError("Can not combine target and bounds")
         elif not any(
-            ("target" in values, "lower_bound" in values, "upper_bound" in values)
+            (
+                (values.get("target") is not None),
+                "lower_bound" in values,
+                "upper_bound" in values,
+            )
         ):
             raise ValueError("Must provide target or lower_bound/upper_bound")
         return values

everest/config/server_config.py

@@ -1,6 +1,4 @@
-import json
 import os
-from pathlib import Path
 from textwrap import dedent
 from typing import Any
 
@@ -12,14 +10,9 @@ from ert.config.queue_config import (
     SlurmQueueOptions,
     TorqueQueueOptions,
 )
-from ert.dark_storage.client import ConnInfo
+from ert.dark_storage.client import ErtClientConnectionInfo
 
-from ..strings import (
-    CERTIFICATE_DIR,
-    DETACHED_NODE_DIR,
-    HOSTFILE_NAME,
-    SESSION_DIR,
-)
+from ..strings import SESSION_DIR
 from .simulator_config import check_removed_config
 
 
@@ -59,21 +52,11 @@ class ServerConfig(BaseModel):
             check_removed_config(data.get("queue_system"))
         return data
 
-    @staticmethod
-    def get_server_url(output_dir: str) -> str:
-        """Return the url of the server.
-
-        If server_info are given, the url is generated using that info. Otherwise
-        server information are retrieved from the hostfile
-        """
-        server_info = ServerConfig.get_server_info(output_dir)
-        return f"https://{server_info['host']}:{server_info['port']}/experiment_server"
-
     @staticmethod
     def get_server_context_from_conn_info(
-        conn_info: ConnInfo,
+        conn_info: ErtClientConnectionInfo,
     ) -> tuple[str, str, tuple[str, str]]:
-        """Get server connection context information from a storage_service ConnInfo.
+        """Get server connection context information from a ErtClientConnectionInfo.
 
         Returns a tuple containing the server URL, certificate file path,
         and authentication credentials.
@@ -83,7 +66,7 @@ class ServerConfig(BaseModel):
         This should be refactored to use the ERT Storage Client class directly instead.
 
         Args:
-            conn_info: An instance of the storage_service client ConnInfo
+            conn_info: An instance of the ErtClientConnectionInfo
 
         Returns:
             tuple: A tuple containing:
@@ -102,34 +85,8 @@ class ServerConfig(BaseModel):
 
         return url, cert_file, auth
 
-    @staticmethod
-    def get_server_info(output_dir: str) -> dict[str, Any]:
-        """Load server information from the hostfile"""
-        host_file_path = Path(ServerConfig.get_hostfile_path(output_dir))
-        if not host_file_path.exists():
-            return {"host": None, "port": None, "cert": None, "auth": None}
-
-        data = json.loads(host_file_path.read_text(encoding="utf-8"))
-
-        if not all(k in data for k in ("host", "port", "cert", "auth")):
-            raise RuntimeError("Malformed hostfile")
-        return data
-
-    @staticmethod
-    def get_detached_node_dir(output_dir: str) -> str:
-        return os.path.join(os.path.abspath(output_dir), DETACHED_NODE_DIR)
-
-    @staticmethod
-    def get_hostfile_path(output_dir: str) -> str:
-        return os.path.join(ServerConfig.get_session_dir(output_dir), HOSTFILE_NAME)
-
     @staticmethod
     def get_session_dir(output_dir: str) -> str:
         """Return path to the session directory containing information about the
         certificates and host information"""
-        return os.path.join(ServerConfig.get_detached_node_dir(output_dir), SESSION_DIR)
-
-    @staticmethod
-    def get_certificate_dir(output_dir: str) -> str:
-        """Return the path to certificate folder"""
-        return os.path.join(ServerConfig.get_session_dir(output_dir), CERTIFICATE_DIR)
+        return os.path.join(os.path.abspath(output_dir), SESSION_DIR)