ert 16.0.9__py3-none-any.whl → 19.0.0rc2__py3-none-any.whl

everest/config/control_config.py

@@ -12,13 +12,12 @@ from typing import (
 from pydantic import AfterValidator, BaseModel, ConfigDict, Field, model_validator
 from ropt.enums import PerturbationType, VariableType
 
-from ert.config import ExtParamConfig
+from ert.config import EverestControl, SamplerConfig
 
 from .control_variable_config import (
     ControlVariableConfig,
     ControlVariableGuessListConfig,
 )
-from .sampler_config import SamplerConfig
 from .validation_utils import (
     control_variables_validation,
     no_dots_in_string,
@@ -167,7 +166,8 @@ class ControlConfig(BaseModel):
             """
         ),
     )
-    perturbation_magnitude: float = Field(
+    perturbation_magnitude: float | None = Field(
+        default=None,
         description=dedent(
             """
             The magnitude of the perturbation of all control variables in the
@@ -186,6 +186,8 @@ class ControlConfig(BaseModel):
               and `min` fields.
 
             This default value can be overridden at the variable level.
+
+            This field is required unless it is explicitly set for all variables.
             """
         ),
     )
@@ -248,6 +250,19 @@ class ControlConfig(BaseModel):
             )
         return values
 
+    @model_validator(mode="after")
+    def validate_perturbation_magnitude(self) -> Self:
+        if self.perturbation_magnitude is None and any(
+            variable
+            for variable in self.variables
+            if variable.perturbation_magnitude is None
+        ):
+            raise ValueError(
+                "A default perturbation_magnitude for all variables must be set "
+                "when not all variables define it explicitly."
+            )
+        return self
+
     @model_validator(mode="after")
     def validate_variables(self) -> Self:
         if self.variables is None:
@@ -274,7 +289,7 @@ class ControlConfig(BaseModel):
     def __hash__(self) -> int:
         return hash(self.name)
 
-    def __eq__(self, other: Any) -> bool:
+    def __eq__(self, other: object) -> bool:
         return isinstance(other, self.__class__) and other.name == self.name
 
     @property
@@ -313,9 +328,69 @@ class ControlConfig(BaseModel):
         extra="forbid",
     )
 
-    def to_ert_parameter_config(self) -> ExtParamConfig:
-        return ExtParamConfig(
+    def to_ert_parameter_config(self) -> EverestControl:
+        types = []
+        initial_guesses = []
+        control_types = []
+        enabled = []
+        mins = []
+        maxs = []
+        perturbation_types = []
+        perturbation_magnitudes = []
+        scaled_ranges = []
+        samplers = []
+
+        def _parse_variable(
+            variable: ControlVariableConfig | ControlVariableGuessListConfig,
+            initial_guess: float | None = None,
+        ) -> None:
+            types.append(self.type)
+            initial_guesses.append(
+                initial_guess
+                if initial_guess is not None
+                else variable.initial_guess
+                if variable.initial_guess is not None
+                else self.initial_guess
+            )
+            control_types.append(self.control_type)
+            enabled.append(
+                variable.enabled if variable.enabled is not None else self.enabled
+            )
+            mins.append(variable.min if variable.min is not None else self.min)
+            maxs.append(variable.max if variable.max is not None else self.max)
+            perturbation_types.append(variable.perturbation_type)
+            perturbation_magnitudes.append(
+                variable.perturbation_magnitude
+                if variable.perturbation_magnitude is not None
+                else self.perturbation_magnitude
+            )
+            scaled_ranges.append(
+                variable.scaled_range
+                if variable.scaled_range is not None
+                else self.scaled_range
+            )
+            samplers.append(variable.sampler or self.sampler)
+
+        for variable in self.variables:
+            if isinstance(variable, ControlVariableConfig):
+                _parse_variable(variable)
+            else:
+                for initial_guess in variable.initial_guess:
+                    _parse_variable(variable, initial_guess=initial_guess)
+
+        return EverestControl(
             name=self.name,
             input_keys=self.formatted_control_names,
+            input_keys_dotdash=self.formatted_control_names_dotdash,
             output_file=self.name + ".json",
+            types=types,
+            initial_guesses=initial_guesses,
+            control_types=control_types,
+            enabled=enabled,
+            min=mins,
+            max=maxs,
+            perturbation_types=perturbation_types,
+            perturbation_magnitudes=perturbation_magnitudes,
+            scaled_ranges=scaled_ranges,
+            samplers=samplers,
         )

everest/config/control_variable_config.py

@@ -12,7 +12,8 @@ from pydantic import (
 )
 from ropt.enums import VariableType
 
-from .sampler_config import SamplerConfig
+from ert.config import SamplerConfig
+
 from .validation_utils import no_dots_in_string, valid_range
 
 
@@ -200,7 +201,7 @@ class ControlVariableConfig(_ControlVariable):
     def __hash__(self) -> int:
         return hash(self.name) + hash(self.index)
 
-    def __eq__(self, other: Any) -> bool:
+    def __eq__(self, other: object) -> bool:
         return (
             isinstance(other, self.__class__)
             and other.name == self.name
@@ -233,7 +234,7 @@ class ControlVariableGuessListConfig(_ControlVariable):
     def __hash__(self) -> int:
         return hash(self.name)
 
-    def __eq__(self, other: Any) -> bool:
+    def __eq__(self, other: object) -> bool:
         return isinstance(other, self.__class__) and other.name == self.name
 
     @property

everest/config/everest_config.py

@@ -2,7 +2,7 @@ import logging
 import os
 from argparse import ArgumentParser
 from copy import copy
-from io import StringIO
+from enum import StrEnum
 from itertools import chain
 from pathlib import Path
 from sys import float_info
@@ -12,6 +12,8 @@ from typing import (
     Any,
     Optional,
     Self,
+    TextIO,
+    TypeVar,
     no_type_check,
 )
 
@@ -27,10 +29,17 @@ from pydantic.json_schema import SkipJsonSchema
 from pydantic_core import ErrorDetails
 from pydantic_core.core_schema import ValidationInfo
 from ruamel.yaml import YAML, YAMLError
-
-from ert.base_model_context import BaseModelWithContextSupport
-from ert.config import ConfigWarning, QueueSystem
-from ert.plugins import ErtPluginContext
+from ruamel.yaml.nodes import ScalarNode
+from ruamel.yaml.representer import Representer
+
+from ert.base_model_context import BaseModelWithContextSupport, use_runtime_plugins
+from ert.config import (
+    ConfigWarning,
+    EverestConstraintsConfig,
+    EverestObjectivesConfig,
+    QueueSystem,
+)
+from ert.plugins import get_site_plugins
 from everest.config.install_template_config import InstallTemplateConfig
 from everest.config.server_config import ServerConfig
 from everest.config.validation_utils import (
@@ -60,7 +69,11 @@ from .forward_model_config import (
 )
 from .input_constraint_config import InputConstraintConfig
 from .install_data_config import InstallDataConfig
-from .install_job_config import InstallJobConfig
+from .install_job_config import (
+    InstallForwardModelStepConfig,
+    InstallJobConfig,
+    InstallWorkflowJobConfig,
+)
 from .model_config import ModelConfig
 from .objective_function_config import ObjectiveFunctionConfig
 from .optimization_config import OptimizationConfig
@@ -125,9 +138,7 @@ def _convert_to_everest_validation_error(
     """
     everest_validation_error = EverestValidationError()
 
-    file_content = []
-    with open(config_path, encoding="utf-8") as f:
-        file_content = f.readlines()
+    file_content = Path(config_path).read_text(encoding="utf-8").splitlines()
 
     for error in validation_error.errors(
         include_context=True, include_input=True, include_url=False
@@ -164,6 +175,9 @@ def _find_loc(loc: tuple[int | str, ...] | None, file_content: list[str]) -> int
     return None
 
 
+InstallJobConfigSubclass = TypeVar("InstallJobConfigSubclass", bound=InstallJobConfig)
+
+
 class EverestConfig(BaseModelWithContextSupport):
     controls: Annotated[list[ControlConfig], AfterValidator(unique_items)] = Field(
         description=dedent(
@@ -177,18 +191,18 @@ class EverestConfig(BaseModelWithContextSupport):
             ```yaml
             controls:
               - name: point
-                  type: generic_control
-                  min: -1.0
+                type: generic_control
+                min: -1.0
+                max: 1.0
+                initial_guess: 0.1
+                perturbation_magnitude: 0.001
+                variables:
+                - name: x
+                  initial_guess: 0.5
+                - name: y
+                  min: 0.0
                   max: 1.0
-                  initial_guess: 0.1
-                  perturbation_magnitude: 0.001
-                  variables:
-                    - name: x
-                      initial_guess: 0.5
-                    - name: y
-                      min: 0.0
-                      max: 1.0
-                    - name: z
+                - name: z
             ```
 
             This defines a group of controls names `point`, with variables `x`,
@@ -210,17 +224,18 @@ class EverestConfig(BaseModelWithContextSupport):
             ```yaml
             controls:
               - name: point
-                  max: 1.0
-                  min: -1.0
-                  initial_guess: 0.25
-                  type: generic_control
-                  variables:
-                    - name: x
-                      index: 1
-                    - name: x
-                      index: 2
-                    - name: x
-                      index: 3
+                max: 1.0
+                min: -1.0
+                initial_guess: 0.25
+                perturbation_magnitude: 0.001
+                type: generic_control
+                variables:
+                  - name: x
+                    index: 1
+                  - name: x
+                    index: 2
+                  - name: x
+                    index: 3
             ```
 
             This defines three variables `point.x.1`, `point.x.2` and
@@ -236,12 +251,13 @@ class EverestConfig(BaseModelWithContextSupport):
             ```yaml
             controls:
               - name: point
-                  max: 1.0
-                  min: -1.0
-                  type: generic_control
-                  variables:
-                    - name: x
-                      initial_guess: [0.25, 0.25, 0.25]
+                max: 1.0
+                min: -1.0
+                perturbation_magnitude: 0.001
+                type: generic_control
+                variables:
+                  - name: x
+                    initial_guess: [0.25, 0.25, 0.25]
             ```
         """
         ),
@@ -365,7 +381,7 @@ class EverestConfig(BaseModelWithContextSupport):
             """
         ),
     )
-    install_jobs: list[InstallJobConfig] = Field(
+    install_jobs: list[InstallForwardModelStepConfig] = Field(
         default_factory=list,
         description=dedent(
             """
@@ -377,7 +393,7 @@ class EverestConfig(BaseModelWithContextSupport):
         ),
         validate_default=True,
     )
-    install_workflow_jobs: list[InstallJobConfig] = Field(
+    install_workflow_jobs: list[InstallWorkflowJobConfig] = Field(
         default_factory=list,
         description=dedent(
             """
@@ -624,7 +640,7 @@ class EverestConfig(BaseModelWithContextSupport):
 
     @model_validator(mode="after")
     def validate_job_executables(self) -> Self:  # pylint: disable=E0213
-        def _check_jobs(jobs: list[InstallJobConfig]) -> list[str]:
+        def _check_jobs(jobs: list[InstallJobConfigSubclass]) -> list[str]:
             errors = []
             for job in jobs:
                 if job.executable is None:
@@ -1017,31 +1033,6 @@ to read summary data from forward model, do:
 
         return the_dict
 
-    @classmethod
-    def with_defaults(cls, **kwargs: Any) -> Self:
-        """
-        Creates an Everest config with default values. Useful for initializing a config
-        without having to provide empty defaults.
-        """
-        defaults = {
-            "controls": [
-                {
-                    "name": "default_group",
-                    "type": "generic_control",
-                    "initial_guess": 0.5,
-                    "perturbation_magnitude": 0.01,
-                    "variables": [
-                        {"name": "default_name", "min": 0, "max": 1},
-                    ],
-                }
-            ],
-            "objective_functions": [{"name": "default"}],
-            "config_path": ".",
-            "model": {"realizations": [0]},
-        }
-
-        return cls.with_plugins({**defaults, **kwargs})  # type: ignore
-
     @staticmethod
     def lint_config_dict(config: ConfigDict) -> list[ErrorDetails]:
         try:
@@ -1073,7 +1064,7 @@ to read summary data from forward model, do:
 
     @classmethod
     def with_plugins(cls, config_dict: dict[str, Any] | ConfigDict) -> Self:
-        with ErtPluginContext():
+        with use_runtime_plugins(get_site_plugins()):
             return cls(**config_dict)
 
     @staticmethod
@@ -1095,26 +1086,58 @@ to read summary data from forward model, do:
         except ValueError as e:
             parser.error(f"Loading config file <{config_path}> failed with: {e}")
 
-    def dump(self, fname: str | None = None) -> str | None:
-        """Write a config dict to file or return it if fname is None."""
-        stripped_conf = self.to_dict()
+    def write_to_file(
+        self, output_file: str | Path | TextIO, drop_config_path: bool = False
+    ) -> None:
+        dictconf = self.to_dict()
+        if drop_config_path:
+            del dictconf["config_path"]
+        if isinstance(output_file, str):
+            output_file = Path(output_file)
+        self.write_dict_to_file(dictconf, output_file)
 
-        del stripped_conf["config_path"]
+    @staticmethod
+    def write_dict_to_file(
+        config: dict[str, Any], output_file: Path | TextIO, safe_and_pure: bool = True
+    ) -> None:
+        yaml = YAML(typ="safe", pure=True) if safe_and_pure else YAML()
+        yaml.indent(mapping=2, sequence=4, offset=2)
+        yaml.preserve_quotes = True
 
-        yaml = YAML(typ="safe", pure=True)
-        yaml.default_flow_style = False
-        if fname is None:
-            with StringIO() as sio:
-                yaml.dump(stripped_conf, sio)
-                return sio.getvalue()
+        def strenum_representer(dumper: Representer, data: StrEnum) -> ScalarNode:
+            return dumper.represent_str(data.value)
 
-        with open(fname, "w", encoding="utf-8") as out:
-            yaml.dump(stripped_conf, out)
+        yaml.representer.add_multi_representer(StrEnum, strenum_representer)
 
-        return None
+        yaml.default_flow_style = False
+        yaml.dump(config, output_file)
 
     @property
     def server_queue_system(self) -> QueueSystem:
         assert self.server is not None
         assert self.server.queue_system is not None
         return self.server.queue_system.name
+
+    def create_ert_objectives_config(self) -> EverestObjectivesConfig:
+        objective_names = [o.name for o in self.objective_functions]
+        return EverestObjectivesConfig(
+            keys=objective_names,
+            input_files=objective_names,
+            weights=[o.weight for o in self.objective_functions],
+            scales=[o.scale for o in self.objective_functions],
+            objective_types=[o.type for o in self.objective_functions],
+        )
+
+    def create_ert_output_constraints_config(self) -> EverestConstraintsConfig | None:
+        if len(self.output_constraints) == 0:
+            return None
+
+        constraint_names = [c.name for c in self.output_constraints]
+        return EverestConstraintsConfig(
+            keys=constraint_names,
+            input_files=constraint_names,
+            scales=[c.scale for c in self.output_constraints],
+            targets=[c.target for c in self.output_constraints],
+            upper_bounds=[c.upper_bound for c in self.output_constraints],
+            lower_bounds=[c.lower_bound for c in self.output_constraints],
+        )

everest/config/forward_model_config.py

@@ -9,7 +9,9 @@ from pydantic import (
 )
 
 from ert.base_model_context import BaseModelWithContextSupport
-from ert.config import ForwardModelStep
+from ert.config import (
+    SiteOrUserForwardModelStep,
+)
 
 
 class ForwardModelResult(BaseModelWithContextSupport):
@@ -74,8 +76,8 @@ class ForwardModelStepConfig(BaseModelWithContextSupport):
         return values
 
     def to_ert_forward_model_step(
-        self, installed_fm_steps: dict[str, ForwardModelStep]
-    ) -> ForwardModelStep:
+        self, installed_fm_steps: dict[str, SiteOrUserForwardModelStep]
+    ) -> SiteOrUserForwardModelStep:
         fm_name, *arglist = self.job.split()
         match fm_name:
             # All three reservoir simulator fm_steps map to

everest/config/install_data_config.py

@@ -9,11 +9,13 @@ from typing import Any
 
 from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
 
-from ert.config import ForwardModelStep
+from ert.config.forward_model_step import (
+    SiteOrUserForwardModelStep,
+)
 
 
 def _is_dir_all_model(source: str, model_realizations: list[int]) -> bool:
-    """Expands <GEO_ID> for all realizations and if:
+    """Expands <REALIZATION_ID> for all realizations and if:
     - all are directories, returns True,
     - all are files, returns False,
     - some are non-existing, raises an AssertionError
@@ -21,7 +23,7 @@ def _is_dir_all_model(source: str, model_realizations: list[int]) -> bool:
 
     is_dir = []
     for model_realization in model_realizations:
-        model_source = source.replace("<GEO_ID>", str(model_realization))
+        model_source = source.replace("<REALIZATION_ID>", str(model_realization))
         if not os.path.exists(model_source):
             msg = (
                 "Expected source to exist for data installation, "
@@ -126,8 +128,8 @@ class InstallDataConfig(BaseModel):
         config_directory: str,
         output_directory: str,
         model_realizations: list[int],
-        installed_fm_steps: dict[str, ForwardModelStep],
-    ) -> ForwardModelStep:
+        installed_fm_steps: dict[str, SiteOrUserForwardModelStep],
+    ) -> SiteOrUserForwardModelStep:
         target = self.target
 
         def _missing_fm_msg(fm_name: str) -> str:

everest/config/install_job_config.py

@@ -5,7 +5,12 @@ from textwrap import dedent
 
 from pydantic import BaseModel, Field, model_validator
 
-from ert.config import ForwardModelStep, forward_model_step_from_config_contents
+from ert.config import (
+    ExecutableWorkflow,
+    UserInstalledForwardModelStep,
+    forward_model_step_from_config_contents,
+    workflow_job_from_file,
+)
 
 
 class InstallJobConfig(BaseModel, extra="forbid"):
@@ -44,12 +49,24 @@ class InstallJobConfig(BaseModel, extra="forbid"):
             raise ValueError("Either source or executable must be provided")
         return self
 
-    def to_ert_forward_model_step(self, config_directory: str) -> ForwardModelStep:
+    @model_validator(mode="after")
+    def validate_source_exists(self) -> InstallJobConfig:
+        if self.source is not None and not Path(self.source).is_file():
+            raise ValueError(f"No such file or directory: {self.source}")
+        return self
+
+
+class InstallForwardModelStepConfig(InstallJobConfig):
+    def to_ert_forward_model_step(
+        self, config_directory: str
+    ) -> UserInstalledForwardModelStep:
         if self.executable is not None:
             executable = Path(self.executable)
             if not executable.is_absolute():
                 executable = Path(config_directory) / executable
-            return ForwardModelStep(name=self.name, executable=str(executable))
+            return UserInstalledForwardModelStep(
+                name=self.name, executable=str(executable)
+            )
         else:
             assert (
                 self.source is not None
@@ -59,3 +76,28 @@ class InstallJobConfig(BaseModel, extra="forbid"):
                 config_file=self.source,
                 name=self.name,
             )
+
+
+class InstallWorkflowJobConfig(InstallJobConfig):
+    def to_ert_executable_workflow(self, config_directory: str) -> ExecutableWorkflow:
+        if self.executable is not None:
+            executable = Path(self.executable)
+            if not executable.is_absolute():
+                executable = Path(config_directory) / executable
+            return ExecutableWorkflow(
+                name=self.name,
+                min_args=None,
+                max_args=None,
+                arg_types=[],
+                executable=str(executable),
+            )
+        else:
+            assert self.source is not None
+            workflow = workflow_job_from_file(
+                config_file=str(Path(config_directory) / self.source),
+                name=self.name,
+                origin="user",
+            )
+            if not isinstance(workflow, ExecutableWorkflow):
+                raise ValueError(f"Workflow must be an executable: {self.source}")
+            return workflow

everest/config/install_template_config.py

@@ -4,7 +4,7 @@ from textwrap import dedent
 
 from pydantic import BaseModel, Field
 
-from ert.config import ForwardModelStep
+from ert.config import SiteOrUserForwardModelStep
 from everest.strings import EVEREST
 
 
@@ -38,9 +38,9 @@ class InstallTemplateConfig(BaseModel, extra="forbid"):
     def to_ert_forward_model_step(
         self,
         control_names: list[str],
-        installed_fm_steps: dict[str, ForwardModelStep],
+        installed_fm_steps: dict[str, SiteOrUserForwardModelStep],
         well_path: str,
-    ) -> ForwardModelStep:
+    ) -> SiteOrUserForwardModelStep:
         fm_step_instance = copy.deepcopy(installed_fm_steps.get("template_render"))
         if fm_step_instance is None:
             raise KeyError(

everest/config/optimization_config.py

@@ -1,11 +1,12 @@
+import importlib
 import logging
 from textwrap import dedent
 from typing import Any, Self
 
 from pydantic import BaseModel, Field, model_validator
+from ropt.workflow import find_optimizer_plugin, validate_optimizer_options
 
 from everest.config.cvar_config import CVaRConfig
-from everest.optimizer.utils import get_ropt_plugin_manager
 from everest.strings import EVEREST
 
 
@@ -346,15 +347,27 @@ class OptimizationConfig(BaseModel, extra="forbid"):
             else f"{self.backend}/{self.algorithm}"
         )
 
-        plugin_manager = get_ropt_plugin_manager()
-        plugin_name = plugin_manager.get_plugin_name("optimizer", algorithm)
+        try:
+            plugin_name = find_optimizer_plugin(algorithm)
+        except ValueError:
+            raise
+        except Exception as exc:
+            ert_version = importlib.metadata.version("ert")
+            ropt_version = importlib.metadata.version("ropt")
+            msg = (
+                f"Error while initializing ropt:\n\n{exc}.\n\n"
+                "There may a be version mismatch between "
+                f"ERT ({ert_version}) and ropt ({ropt_version})\n"
+                "If the installation is correct, please report this as a bug."
+            )
+            raise RuntimeError(msg) from exc
         if plugin_name is None:
             raise ValueError(f"Optimizer algorithm '{algorithm}' not found")
         self._optimization_plugin_name = plugin_name
 
-        plugin_manager.get_plugin("optimizer", algorithm).validate_options(
-            self.algorithm, self.options or self.backend_options
-        )
+        options = self.options or self.backend_options
+        if options:
+            validate_optimizer_options(algorithm, options)
 
         self.backend = None
         self.algorithm = algorithm

everest/config/output_constraint_config.py

@@ -74,10 +74,16 @@ class OutputConstraintConfig(BaseModel, extra="forbid"):
     @model_validator(mode="before")
     @classmethod
     def validate_target_or_bounds(cls, values: dict[str, Any]) -> dict[str, Any]:
-        if "target" in values and ("lower_bound" in values or "upper_bound" in values):
+        if (values.get("target") is not None) and (
+            "lower_bound" in values or "upper_bound" in values
+        ):
             raise ValueError("Can not combine target and bounds")
         elif not any(
-            ("target" in values, "lower_bound" in values, "upper_bound" in values)
+            (
+                (values.get("target") is not None),
+                "lower_bound" in values,
+                "upper_bound" in values,
+            )
         ):
             raise ValueError("Must provide target or lower_bound/upper_bound")
         return values