ert 17.1.9__py3-none-any.whl → 18.0.1__py3-none-any.whl

everest/config/utils.py

@@ -1,108 +1,28 @@
-from copy import deepcopy
-from typing import Any, Literal
+from ert.config import ExtParamConfig, SamplerConfig
 
-from .control_config import ControlConfig
-from .control_variable_config import ControlVariableGuessListConfig
-from .sampler_config import SamplerConfig
 
-
-class FlattenedControls:
-    def __init__(self, controls: list[ControlConfig]) -> None:
-        control_dicts = _get_control_dicts(controls)
-        self.names = [control["name"] for control in control_dicts]
-        self.types: list[Literal["real", "integer"]] = [
-            control["control_type"] for control in control_dicts
-        ]
-        self.initial_guesses = [control["initial_guess"] for control in control_dicts]
-        self.lower_bounds = [control["min"] for control in control_dicts]
-        self.upper_bounds = [control["max"] for control in control_dicts]
-        self.scaled_ranges = [control["scaled_range"] for control in control_dicts]
-        self.enabled = [control["enabled"] for control in control_dicts]
-        self.perturbation_magnitudes = [
-            control["perturbation_magnitude"] for control in control_dicts
-        ]
-        self.perturbation_types = [
-            control["perturbation_type"] for control in control_dicts
-        ]
-        self.samplers, self.sampler_indices = _get_samplers(controls)
-
-
-def _get_control_dicts(controls: list[ControlConfig]) -> list[dict[str, Any]]:
-    def _inject_defaults(control: ControlConfig, var_dict: dict[str, Any]) -> None:
-        for key in [
-            "type",
-            "initial_guess",
-            "control_type",
-            "enabled",
-            "min",
-            "max",
-            "perturbation_type",
-            "perturbation_magnitude",
-            "scaled_range",
-        ]:
-            if var_dict.get(key) is None:
-                var_dict[key] = getattr(control, key)
-
-    control_dicts: list[dict[str, Any]] = []
-    for control in controls:
-        for variable in control.variables:
-            if isinstance(variable, ControlVariableGuessListConfig):
-                for index, guess in enumerate(variable.initial_guess, start=1):
-                    var_dict = deepcopy(variable.model_dump())
-                    var_dict["name"] = (control.name, variable.name, index)
-                    var_dict["initial_guess"] = guess
-                    _inject_defaults(control, var_dict)
-                    control_dicts.append(var_dict)
-            else:
-                var_dict = deepcopy(variable.model_dump())
-                var_dict["name"] = (
-                    (control.name, variable.name)
-                    if variable.index is None
-                    else (control.name, variable.name, variable.index)
-                )
-                _inject_defaults(control, var_dict)
-                control_dicts.append(var_dict)
-    return control_dicts
-
-
-def _get_samplers(
-    controls: list[ControlConfig],
+def get_samplers(
+    controls: list[ExtParamConfig],
 ) -> tuple[list[SamplerConfig | None], list[int]]:
-    samplers: list[SamplerConfig | None] = []
-    sampler_indices: list[int] = []
-
-    default_sampler_index: int | None = None
-
-    for control in controls:
-        control_sampler_index: int | None = None
-
-        for variable in control.variables:
-            if variable.sampler is not None:
-                # Use the sampler of the variable:
-                samplers.append(variable.sampler)
-                variable_sampler_index = len(samplers) - 1
-            elif control.sampler is not None:
-                # Use the sampler of the control:
-                if control_sampler_index is None:
-                    samplers.append(control.sampler)
-                    control_sampler_index = len(samplers) - 1
-                variable_sampler_index = control_sampler_index
-            else:
-                # Use the default sampler:
-                if default_sampler_index is None:
-                    samplers.append(None)
-                    default_sampler_index = len(samplers) - 1
-                variable_sampler_index = default_sampler_index
-
-            if isinstance(variable, ControlVariableGuessListConfig):
-                sampler_indices.extend(
-                    [variable_sampler_index] * len(variable.initial_guess)
-                )
-            else:
-                sampler_indices.append(variable_sampler_index)
-
-    return samplers, sampler_indices
-
-
-def flatten_controls(controls: list[ControlConfig]) -> FlattenedControls:
-    return FlattenedControls(controls)
+    """
+    Create a list of unique samplers, and a list mapping variable index
+    to sampler index. I.e., this points each control variable to
+    a sampler by index.
+    """
+    flattened_samplers = [
+        sampler for control in controls for sampler in control.samplers
+    ]
+    unique_samplers: list[SamplerConfig | None] = []
+    variable_to_unique_sampler_index: list[int] = []
+    for sampler in flattened_samplers:
+        try:
+            unique_sampler_index = next(
+                i for i, s in enumerate(unique_samplers) if s == sampler
+            )
+        except StopIteration:
+            unique_sampler_index = len(unique_samplers)
+            unique_samplers.append(sampler)
+
+        variable_to_unique_sampler_index.append(unique_sampler_index)
+
+    return unique_samplers, variable_to_unique_sampler_index

everest/config/validation_utils.py

@@ -72,15 +72,15 @@ class InstallDataContext:
                 source = item.source
                 target = item.target
 
-            if "<GEO_ID>" not in source:
+            if "<REALIZATION_ID>" not in source:
                 self._set_symlink(source, target, None)
 
         return self
 
     def _set_symlink(self, source: str, target: str, realization: int | None) -> None:
         if realization is not None:
-            source = source.replace("<GEO_ID>", str(realization))
-            target = target.replace("<GEO_ID>", str(realization))
+            source = source.replace("<REALIZATION_ID>", str(realization))
+            target = target.replace("<REALIZATION_ID>", str(realization))
 
         tmp_target = Path(self._temp_dir.name) / Path(target)
         if tmp_target.exists():
@@ -101,7 +101,7 @@ class InstallDataContext:
 
     def add_links_for_realization(self, realization: int) -> None:
         for data in self._install_data:
-            if data.source is not None and "<GEO_ID>" in data.source:
+            if data.source is not None and "<REALIZATION_ID>" in data.source:
                 self._set_symlink(data.source, data.target, realization)
 
     def __exit__(self, exc_type: Any, exc_value: Any, exc_tb: Any) -> None:
@@ -190,7 +190,7 @@ def unique_items(items: Sequence[T]) -> Sequence[T]:
 
 
 def valid_range(range_value: tuple[float, float]) -> tuple[float, float]:
-    if range_value[0] >= range_value[1]:
+    if range_value is not None and range_value[0] >= range_value[1]:
         raise ValueError("scaled_range must be a valid range [a, b], where a < b.")
     return range_value
 
@@ -242,8 +242,8 @@ def as_abs_path(path: str, config_dir: str) -> str:
 
 
 def expand_model_id_paths(path_source: str, realizations: list[int]) -> list[str]:
-    if "<GEO_ID>" in path_source:
-        return [path_source.replace("<GEO_ID>", str(r)) for r in realizations]
+    if "<REALIZATION_ID>" in path_source:
+        return [path_source.replace("<REALIZATION_ID>", str(r)) for r in realizations]
     return [path_source]
 
 
@@ -251,7 +251,8 @@ def check_path_exists(
     path_source: str, config_path: Path | None, realizations: list[int]
 ) -> None:
     """Check if the given path exists. If the given path contains <CONFIG_PATH>
-    or GEO_ID they will be expanded and all instances of expanded paths need to exist.
+    or REALIZATION_ID they will be expanded and all instances of expanded paths
+    need to exist.
     """
     if not isinstance(path_source, str):
         raise ValueError(
@@ -285,8 +286,7 @@ def check_writeable_path(path_source: str, config_path: Path) -> None:
             if os.access(path, os.W_OK | os.X_OK):
                 break
         elif os.path.isfile(path):
-            # path is a file, cannot create folder
-            raise ValueError(f"File {path} exists, cannot create path {path_source}")
+            raise ValueError(f"{path} is a file, cannot create folders inside it")
         parent = os.path.dirname(path)
         if parent == path:  # ie, if path is root
             break

everest/config_file_loader.py

@@ -26,7 +26,7 @@ SUBSTITUTION_PATTERN = r"(r\{\{.*?\}\})"
 
 # Jinja vars which should NOT be included in definitions portion of config.
 ERT_CONFIG_TEMPLATES = {
-    "realization": "GEO_ID",
+    "realization": "REALIZATION_ID",
     "runpath_file": "RUNPATH_FILE",
 }
 
@@ -67,7 +67,7 @@ def _get_definitions(
                     f"Internal key {key} specified by user as {defs[key]}. "
                     f"Overriding as {val}"
                 )
-            defs[key] = f"<{val}>"  # ert uses <GEO_ID> as format
+            defs[key] = f"<{val}>"  # ert uses <REALIZATION_ID> as format
     else:
         logging.getLogger(EVEREST).warning("Empty configuration file provided!")
 
@@ -160,6 +160,17 @@ def yaml_file_to_substituted_config_dict(config_path: str) -> dict[str, Any]:
     # Replace in definitions
     config = jenv.from_string(txt).render(**definitions)
 
+    # Handle deprecated use of <GEO_ID>:
+    config, count = re.subn(
+        rf"<\s*{re.escape('GEO_ID')}\s*>", "<REALIZATION_ID>", config
+    )
+    if count > 0:
+        message = r"<GEO_ID> is deprecated, please replace with 'r{{realization}}'."
+        print(message)
+        logging.getLogger(EVEREST).warning(
+            "Deprecated key <GEO_ID> replaced by <REALIZATION_ID>."
+        )
+
     # Load the config with definitions again as yaml
     yaml = YAML(typ="safe", pure=True).load(config)
 

everest/detached/everserver.py

@@ -12,7 +12,7 @@ import yaml
 from opentelemetry.trace.propagation.tracecontext import TraceContextTextMapPropagator
 
 from ert.plugins.plugin_manager import ErtPluginManager
-from ert.services import StorageService
+from ert.services import ErtServer
 from ert.services._base_service import BaseServiceExit
 from ert.storage import ExperimentStatus
 from ert.storage.local_experiment import ExperimentState
@@ -31,7 +31,7 @@ logger = logging.getLogger(__name__)
 
 
 def _configure_loggers(
-    detached_dir: Path, log_dir: Path, logging_level: int, output_file: str | None
+    log_dir: Path, logging_level: int, output_file: str | None
 ) -> None:
     def make_handler_config(path: Path, log_level: int) -> dict[str, Any]:
         makedirs_if_needed(str(path.parent))
@@ -46,7 +46,7 @@ def _configure_loggers(
         "version": 1,
         "handlers": {
             "endpoint_log": make_handler_config(
-                detached_dir / "endpoint.log", logging_level
+                log_dir / "everserver.log", logging_level
             ),
             "everest_log": make_handler_config(log_dir / "everest.log", logging_level),
             "forward_models_log": make_handler_config(
@@ -151,7 +151,6 @@ def main() -> None:
     ):
         try:
             _configure_loggers(
-                detached_dir=Path(ServerConfig.get_detached_node_dir(output_dir)),
                 log_dir=Path(output_dir) / OPTIMIZATION_LOG_DIR,
                 logging_level=options.logging_level,
                 output_file=log_file.name,
@@ -163,11 +162,11 @@ def main() -> None:
             # Starting the server
             server_path = os.path.abspath(ServerConfig.get_session_dir(output_dir))
             status = ""
-            with StorageService.init_service(
-                timeout=240, project=server_path, logging_config=log_file.name
+            with ErtServer.init_service(
+                timeout=240, project=Path(server_path), logging_config=log_file.name
             ) as server:
-                server.fetch_conn_info()
-                with StorageService.session(project=Path(server_path)) as client:
+                server.fetch_connection_info()
+                with ErtServer.session(project=Path(server_path)) as client:
                     done = False
                     while not done:
                         response = client.get(

everest/everest_storage.py

@@ -12,8 +12,7 @@ import numpy as np
 import polars as pl
 from ropt.results import FunctionResults, GradientResults, Results
 
-from ert.config import EverestObjectivesConfig
-from everest.config.output_constraint_config import OutputConstraintConfig
+from ert.config import EverestConstraintsConfig, EverestObjectivesConfig
 from everest.strings import EVEREST
 
 logger = logging.getLogger(__name__)
@@ -532,7 +531,7 @@ class EverestStorage:
         self,
         formatted_control_names: list[str],
         objective_functions: EverestObjectivesConfig,
-        output_constraints: list[OutputConstraintConfig] | None,
+        output_constraints: EverestConstraintsConfig | None,
         realizations: list[int],
     ) -> None:
         controls = pl.DataFrame(
@@ -566,13 +565,7 @@ class EverestStorage:
         )
 
         nonlinear_constraints = (
-            pl.DataFrame(
-                {
-                    "constraint_name": [
-                        constraint.name for constraint in output_constraints
-                    ],
-                }
-            )
+            pl.DataFrame({"constraint_name": output_constraints.keys})
             if output_constraints
             else None
         )
@@ -1001,6 +994,9 @@ class EverestStorage:
     def export_dataframes(
         self,
     ) -> tuple[pl.DataFrame, pl.DataFrame, pl.DataFrame]:
+        if not self.data.batches:
+            return (pl.DataFrame(), pl.DataFrame(), pl.DataFrame())
+
         batch_dfs_to_join = {}  # type: ignore
         realization_dfs_to_join = {}  # type: ignore
         perturbation_dfs_to_join = {}  # type: ignore

everest/gui/everest_client.py

@@ -128,7 +128,6 @@ class EverestClient:
             supports_rerunning_failed_realizations=False,
             start_simulations_thread=start_fn,
             cancel=self.stop,
-            get_runtime=self.get_runtime,
             has_failed_realizations=lambda: False,
         )
 

everest/gui/main_window.py

@@ -13,7 +13,7 @@ from PyQt6.QtWidgets import (
 from ert.gui.ertnotifier import ErtNotifier
 from ert.gui.simulation.run_dialog import RunDialog
 from ert.plugins import ErtPluginManager
-from ert.services import StorageService
+from ert.services import ErtServer
 from everest.config import ServerConfig
 from everest.detached import wait_for_server
 from everest.gui.everest_client import EverestClient
@@ -43,7 +43,7 @@ class EverestMainWindow(QMainWindow):
         self.setCentralWidget(self.central_widget)
 
     def run(self) -> None:
-        storage_client = StorageService.session(
+        storage_client = ErtServer.session(
             Path(ServerConfig.get_session_dir(self.output_dir))
         )
         wait_for_server(storage_client, 60)

everest/optimizer/everest2ropt.py

@@ -3,34 +3,48 @@ from typing import Any
 
 from ropt.enums import PerturbationType, VariableType
 
-from ert.config import EverestObjectivesConfig
+from ert.config import EverestConstraintsConfig, EverestObjectivesConfig, ExtParamConfig
 from everest.config import (
-    ControlConfig,
     InputConstraintConfig,
     ModelConfig,
     OptimizationConfig,
     OutputConstraintConfig,
 )
-from everest.config.utils import FlattenedControls
+from everest.config.utils import get_samplers
 
 
 def _parse_controls(
-    controls: FlattenedControls, random_seed: int
+    controls: list[ExtParamConfig], random_seed: int
 ) -> tuple[dict[str, Any], list[dict[str, Any]] | None]:
-    control_types = [VariableType[type_.upper()] for type_ in controls.types]
+    control_types = [
+        VariableType[type_.upper()]
+        for control in controls
+        for type_ in control.control_types
+    ]
+    initial_guesses = [
+        initial_guess
+        for control in controls
+        for initial_guess in control.initial_guesses
+    ]
+    samplers, sampler_indices = get_samplers(controls)
     ropt_variables: dict[str, Any] = {
         "types": None if all(item is None for item in control_types) else control_types,
-        "variable_count": len(controls.initial_guesses),
-        "lower_bounds": controls.lower_bounds,
-        "upper_bounds": controls.upper_bounds,
+        "variable_count": len(initial_guesses),
+        "lower_bounds": [min_ for control in controls for min_ in control.min],
+        "upper_bounds": [max_ for control in controls for max_ in control.max],
         "perturbation_types": [
             PerturbationType[perturbation_type.upper()]
-            for perturbation_type in controls.perturbation_types
+            for control in controls
+            for perturbation_type in control.perturbation_types
+        ],
+        "perturbation_magnitudes": [
+            perturbation_magnitude
+            for control in controls
+            for perturbation_magnitude in control.perturbation_magnitudes
         ],
-        "perturbation_magnitudes": controls.perturbation_magnitudes,
-        "mask": controls.enabled,
+        "mask": [enabled for control in controls for enabled in control.enabled],
         "seed": random_seed,
-        "samplers": controls.sampler_indices,
+        "samplers": sampler_indices,
     }
 
     ropt_samplers = [
@@ -41,7 +55,7 @@ def _parse_controls(
             "options": {} if sampler.options is None else sampler.options,
             "shared": False if sampler.shared is None else sampler.shared,
         }
-        for sampler in controls.samplers
+        for sampler in samplers
     ]
 
     return ropt_variables, ropt_samplers
@@ -102,13 +116,13 @@ def _get_bounds(
 
 def _parse_input_constraints(
     input_constraints: list[InputConstraintConfig],
-    controls: list[ControlConfig],
+    controls: list[ExtParamConfig],
 ) -> dict[str, Any]:
     formatted_control_names = [
-        name for config in controls for name in config.formatted_control_names
+        name for config in controls for name in config.input_keys
     ]
     formatted_control_names_dotdash = [
-        name for config in controls for name in config.formatted_control_names_dotdash
+        name for config in controls for name in config.input_keys_dotdash
     ]
 
     def _get_control_index(name: str) -> int:
@@ -143,13 +157,26 @@ def _parse_input_constraints(
 
 
 def _parse_output_constraints(
-    output_constraints: list[OutputConstraintConfig],
+    output_constraints: EverestConstraintsConfig | None,
 ) -> dict[str, Any]:
     if output_constraints:
-        lower_bounds, upper_bounds = _get_bounds(output_constraints)
         return {
-            "lower_bounds": lower_bounds,
-            "upper_bounds": upper_bounds,
+            "lower_bounds": [
+                target if target is not None else lb
+                for lb, target in zip(
+                    output_constraints.lower_bounds,
+                    output_constraints.targets,
+                    strict=False,
+                )
+            ],
+            "upper_bounds": [
+                target if target is not None else ub
+                for ub, target in zip(
+                    output_constraints.upper_bounds,
+                    output_constraints.targets,
+                    strict=False,
+                )
+            ],
         }
     return {}
 
@@ -248,18 +275,16 @@ def _parse_optimization(
 
 
 def everest2ropt(
-    controls: list[ControlConfig],
+    controls: list[ExtParamConfig],
     objective_functions: EverestObjectivesConfig,
     input_constraints: list[InputConstraintConfig],
-    output_constraints: list[OutputConstraintConfig],
+    output_constraints: EverestConstraintsConfig | None,
     optimization: OptimizationConfig | None,
     model: ModelConfig,
     random_seed: int,
     optimization_output_dir: str,
 ) -> tuple[dict[str, Any], list[float]]:
-    flattened_controls = FlattenedControls(controls)
-
-    ropt_variables, ropt_samplers = _parse_controls(flattened_controls, random_seed)
+    ropt_variables, ropt_samplers = _parse_controls(controls, random_seed)
     ropt_objectives, ropt_function_estimators = _parse_objectives(objective_functions)
     ropt_linear_constraints = _parse_input_constraints(input_constraints, controls)
     ropt_nonlinear_constraints = _parse_output_constraints(output_constraints)
@@ -293,13 +318,11 @@ def everest2ropt(
         "realizations": ropt_realizations,
         "optimizer": ropt_optimizer,
         "names": {
-            "variable": [
-                name for config in controls for name in config.formatted_control_names
-            ],
+            "variable": [name for config in controls for name in config.input_keys],
             "objective": objective_functions.keys,
-            "nonlinear_constraint": [
-                constraint.name for constraint in output_constraints
-            ],
+            "nonlinear_constraint": output_constraints.keys
+            if output_constraints is not None
+            else [],
             "realization": model.realizations,
         },
     }
@@ -316,4 +339,8 @@ def everest2ropt(
     if ropt_samplers:
         ropt_config["samplers"] = ropt_samplers
 
-    return ropt_config, flattened_controls.initial_guesses
+    return ropt_config, [
+        initial_guess
+        for control in controls
+        for initial_guess in control.initial_guesses
+    ]

everest/optimizer/opt_model_transforms.py

@@ -11,14 +11,11 @@ from ropt.transforms.base import (
     VariableTransform,
 )
 
-from ert.config import EverestObjectivesConfig
+from ert.config import EverestConstraintsConfig, EverestObjectivesConfig, ExtParamConfig
 from everest.config import (
-    ControlConfig,
     InputConstraintConfig,
     ModelConfig,
-    OutputConstraintConfig,
 )
-from everest.config.utils import FlattenedControls
 
 
 class EverestOptModelTransforms(TypedDict):
@@ -429,19 +426,22 @@ class ConstraintScaler(NonLinearConstraintTransform):
 
 
 def get_optimization_domain_transforms(
-    controls: list[ControlConfig],
+    controls: list[ExtParamConfig],
     objectives: EverestObjectivesConfig,
     input_constraints: list[InputConstraintConfig] | None,
-    output_constraints: list[OutputConstraintConfig] | None,
+    output_constraints: EverestConstraintsConfig | None,
     model: ModelConfig,
     auto_scale: bool,
 ) -> EverestOptModelTransforms:
-    flattened_controls = FlattenedControls(controls)
     control_scaler = ControlScaler(
-        flattened_controls.lower_bounds,
-        flattened_controls.upper_bounds,
-        flattened_controls.scaled_ranges,
-        flattened_controls.types,
+        [min_ for control in controls for min_ in control.min],
+        [max_ for control in controls for max_ in control.max],
+        [
+            scaled_range
+            for control in controls
+            for scaled_range in control.scaled_ranges
+        ],
+        [type_ for control in controls for type_ in control.control_types],
         auto_scale_input_constraints=auto_scale,
         input_constraint_scales=(
             None
@@ -472,8 +472,7 @@ def get_optimization_domain_transforms(
         ConstraintScaler(
             auto_scale=auto_scale,
             scales=[
-                1.0 if constraint.scale is None else constraint.scale
-                for constraint in output_constraints
+                1.0 if scale is None else scale for scale in output_constraints.scales
             ],
             realization_weights=realization_weights,
         )

everest/optimizer/utils.py

@@ -1,29 +0,0 @@
-import importlib
-
-from ropt.plugins import PluginManager
-
-
-def get_ropt_plugin_manager() -> PluginManager:
-    # To check the validity of optimization and sampler backends and their
-    # supported algorithms or methods, an instance of a ropt PluginManager is
-    # needed. Everest also needs a ropt plugin manager at runtime which may add
-    # additional optimization and/or sampler backends. To be sure that these
-    # added backends are detected, all code should use this function to access
-    # the plugin manager. Any optimizer/sampler plugins that need to be added at
-    # runtime should be added in this function.
-    #
-    # Note: backends can also be added via the Python entrypoints mechanism,
-    # these are detected by default and do not need to be added here.
-
-    try:
-        return PluginManager()
-    except Exception as exc:
-        ert_version = importlib.metadata.version("ert")
-        ropt_version = importlib.metadata.version("ropt")
-        msg = (
-            f"Error while initializing ropt:\n\n{exc}.\n\n"
-            "Check the everest installation, there may a be version mismatch.\n"
-            f"  (ERT: {ert_version}, ropt: {ropt_version})\n"
-            "If the everest installation is correct, please report this as a bug."
-        )
-        raise RuntimeError(msg) from exc

everest/strings.py

@@ -1,8 +1,5 @@
 from enum import StrEnum
 
-CERTIFICATE_DIR = "cert"
-
-DETACHED_NODE_DIR = "detached_node_output"
 DEFAULT_OUTPUT_DIR = "everest_output"
 DEFAULT_LOGGING_FORMAT = "%(asctime)s %(name)s %(levelname)s: %(message)s"
 
@@ -10,8 +7,6 @@ EVEREST = "everest"
 EVERSERVER = "everserver"
 EXPERIMENT_SERVER = "experiment_server"
 
-HOSTFILE_NAME = "storage_server.json"
-
 NAME = "name"
 
 OPTIMIZATION_OUTPUT_DIR = "optimization_output"