ert 16.0.2__py3-none-any.whl → 17.0.0__py3-none-any.whl

everest/config/simulator_config.py

@@ -75,9 +75,22 @@ class SimulatorConfig(BaseModelWithContextSupport, extra="forbid"):
         default=None,
         description=dedent(
             """
-            Maximum allowed memory usage of a forward model.
+            Amount of memory to set aside for a forward model.
 
-            When set, a job is only allowed to use `max_memory` of memory.
+            This information is propagated to the queue system as the amount of memory
+            to reserve/book for a realization to complete. It is up to the configuration
+            of the queuing system how to treat this information, but usually it will
+            stop more realizations being assigned to a compute node if the compute nodes
+            memory is already fully booked.
+
+            Setting this number lower than the peak memory consumption of each
+            realization puts the realization at risk of being killed in an out-of-memory
+            situation. Setting this number higher than needed will give longer wait
+            times in the queue.
+
+            For the local queue system, this keyword has no effect. In that scenario,
+            you can use `max_running`  to choke the memory consumption.
+            scheduling of compute jobs.
 
             `max_memory` may be an integer value, indicating the number of
             bytes, or a string consisting of a number followed by a unit. The
@@ -94,9 +107,6 @@ class SimulatorConfig(BaseModelWithContextSupport, extra="forbid"):
             Spaces between the number and the unit are ignored, and so are any
             characters after the first. For example: 2g, 2G, and 2 GB all
             resolve to the same value: 2 gigabytes, equaling 2 * 1024**3 bytes.
-
-            If not set, or a set to zero, the allowed amount of memory is
-            unlimited.
             """
         ),
     )
@@ -132,17 +142,40 @@ class SimulatorConfig(BaseModelWithContextSupport, extra="forbid"):
         ),
     )
 
-    @field_validator("queue_system", mode="before")
+    @model_validator(mode="before")
     @classmethod
-    def default_local_queue(cls, v: Any, info: ValidationInfo) -> Any:
-        if v is None:
+    def apply_site_or_default_queue_if_no_user_queue(
+        cls, data: dict[str, Any], info: ValidationInfo
+    ) -> Any:
+        queue_system = data.get("queue_system")
+        if queue_system is None:
             options = None
             if info.context:
                 options = info.context.queue_options
-            return options or LocalQueueOptions(max_running=8)
-        return v
 
-    @field_validator("max_memory")
+            defaulted_queue_options = (
+                options.model_dump()
+                if options is not None
+                else LocalQueueOptions(max_running=8).model_dump()
+            )
+
+            user_configured_max_memory = data.get("max_memory")
+            if user_configured_max_memory is not None:
+                cls.validate_max_memory(max_memory=user_configured_max_memory)
+                defaulted_queue_options["realization_memory"] = (
+                    user_configured_max_memory
+                )
+
+            user_configured_cores_per_node = data.get("cores_per_node")
+            if user_configured_cores_per_node is not None:
+                defaulted_queue_options["num_cpu"] = user_configured_cores_per_node
+
+            data["queue_system"] = defaulted_queue_options
+            data["max_memory"] = None
+
+        return data
+
+    @field_validator("max_memory", mode="before")
     @classmethod
     def validate_max_memory(cls, max_memory: int | str | None) -> str | None:
         if max_memory is None:
@@ -163,14 +196,26 @@ class SimulatorConfig(BaseModelWithContextSupport, extra="forbid"):
 
     @model_validator(mode="after")
     def update_max_memory(config: "SimulatorConfig") -> "SimulatorConfig":
+        if config.max_memory is None:
+            return config
+        parsed_max_memory = (
+            parse_string_to_bytes(config.max_memory)
+            if type(config.max_memory) is str
+            else int(config.max_memory)
+        )
         if (
-            config.max_memory is not None
-            and config.queue_system is not None
+            config.queue_system is not None
             and config.queue_system.realization_memory == 0
         ):
-            config.queue_system.realization_memory = (
-                parse_string_to_bytes(config.max_memory)
-                if type(config.max_memory) is str
-                else int(config.max_memory)
+            config.queue_system.realization_memory = parsed_max_memory
+        elif (
+            config.queue_system is not None
+            and config.queue_system.realization_memory > 0
+            and config.queue_system.realization_memory != parsed_max_memory
+        ):
+            raise ConfigValidationError(
+                "Ambiguous configuration of realization_memory. "
+                "Specify either max_memory or realization_memory, not both"
             )
+
         return config

everest/config/validation_utils.py

@@ -15,7 +15,7 @@ from everest.util.forward_models import (
     parse_forward_model_file,
 )
 
-from .install_job_config import InstallJobConfig
+from .install_job_config import InstallForwardModelStepConfig
 
 _VARIABLE_ERROR_MESSAGE = (
     "Variable {name} must define {variable_type} value either"
@@ -39,6 +39,8 @@ _RESERVED_WORDS = [
     "total_objective_value",
 ]
 
+_OVERWRITE_MESSAGE = "Are you overwriting other parts of install_data?"
+
 
 class InstallDataContext:
     def __init__(
@@ -82,9 +84,20 @@ class InstallDataContext:
 
         tmp_target = Path(self._temp_dir.name) / Path(target)
         if tmp_target.exists():
+            if tmp_target.is_dir() and not tmp_target.is_symlink():
+                raise ValueError(
+                    "Cannot make symlink due to existing directory at target location"
+                    f" {tmp_target}. " + _OVERWRITE_MESSAGE
+                )
             tmp_target.unlink()
-        tmp_target.parent.mkdir(parents=True, exist_ok=True)
-        tmp_target.symlink_to(as_abs_path(source, self._config_dir))
+        try:
+            tmp_target.parent.mkdir(parents=True, exist_ok=True)
+            tmp_target.symlink_to(as_abs_path(source, self._config_dir))
+        except FileExistsError as err:
+            raise ValueError(
+                f"Cannot install data {source} into {target} due to existing "
+                "file or directory. " + _OVERWRITE_MESSAGE
+            ) from err
 
     def add_links_for_realization(self, realization: int) -> None:
         for data in self._install_data:
@@ -284,7 +297,7 @@ def check_writeable_path(path_source: str, config_path: Path) -> None:
 
 
 def validate_forward_model_configs(
-    forward_model: list[str], install_jobs: list[InstallJobConfig]
+    forward_model: list[str], install_jobs: list[InstallForwardModelStepConfig]
 ) -> None:
     if not forward_model:
         return

everest/config_file_loader.py

@@ -4,6 +4,7 @@ import logging
 import os
 import re
 from io import StringIO
+from pathlib import Path
 from typing import Any
 
 import jinja2
@@ -31,23 +32,22 @@ ERT_CONFIG_TEMPLATES = {
 
 
 def load_yaml(file_name: str, safe: bool = False) -> dict[str, Any]:
-    with open(file_name, encoding="utf-8") as input_file:
-        input_data: list[str] = input_file.readlines()
-        try:
-            yaml = YAML(typ="safe", pure=True) if safe else YAML()
-            yaml.preserve_quotes = True
-            return yaml.load("".join(input_data))
-        except YAMLError as exc:
-            if hasattr(exc, "problem_mark"):
-                mark = exc.problem_mark
-                raise YAMLError(
-                    str(exc)
-                    + "\nError in line: {}\n {}^)".format(
-                        input_data[mark.line], " " * mark.column
-                    )
-                ) from exc
-            else:
-                raise YAMLError(str(exc)) from exc
+    input_data = Path(file_name).read_text(encoding="utf-8").splitlines()
+    try:
+        yaml = YAML(typ="safe", pure=True) if safe else YAML()
+        yaml.preserve_quotes = True
+        return yaml.load("\n".join(input_data))
+    except YAMLError as exc:
+        if hasattr(exc, "problem_mark"):
+            mark = exc.problem_mark
+            raise YAMLError(
+                str(exc)
+                + "\nError in line: {}\n {}^)".format(
+                    input_data[mark.line], " " * mark.column
+                )
+            ) from exc
+        else:
+            raise YAMLError(str(exc)) from exc
 
 
 def _get_definitions(

everest/detached/__init__.py

@@ -4,28 +4,22 @@ Client methods for interacting with everserver
 
 from .client import (
     PROXY,
-    ExperimentState,
-    everserver_status,
     server_is_running,
     start_experiment,
     start_monitor,
     start_server,
     stop_server,
-    update_everserver_status,
     wait_for_server,
     wait_for_server_to_stop,
 )
 
 __all__ = [
     "PROXY",
-    "ExperimentState",
-    "everserver_status",
     "server_is_running",
     "start_experiment",
     "start_monitor",
     "start_server",
     "stop_server",
-    "update_everserver_status",
     "wait_for_server",
     "wait_for_server_to_stop",
 ]

everest/detached/client.py

@@ -1,7 +1,5 @@
 import asyncio
-import json
 import logging
-import os
 import re
 import ssl
 import time
@@ -17,9 +15,7 @@ from websockets import ConnectionClosedError, ConnectionClosedOK
 from websockets.sync.client import connect
 
 from ert.dark_storage.client import Client
-from ert.ensemble_evaluator import EndEvent
 from ert.run_models.event import EverestBatchResultEvent, status_event_from_json
-from ert.run_models.run_model import ExperimentState
 from ert.scheduler import create_driver
 from ert.scheduler.driver import Driver, FailedSubmit
 from ert.scheduler.event import StartedEvent
@@ -49,7 +45,7 @@ async def start_server(config: EverestConfig, logging_level: int) -> Driver:
     """
     Start an Everest server running the optimization defined in the config
     """
-    driver = create_driver(config.server.queue_system)  # type: ignore
+    driver = create_driver(config.server.queue_system, poll_period=0.1)  # type: ignore
     try:
         args = [
             "--output-dir",
@@ -236,9 +232,7 @@ def start_monitor(
                 try:
                     message = websocket.recv(timeout=1.0)
                     event = status_event_from_json(message)
-                    if isinstance(event, EndEvent):
-                        print(event.msg)
-                    elif isinstance(event, EverestBatchResultEvent):
+                    if isinstance(event, EverestBatchResultEvent):
                         if event.result_type == "FunctionResult":
                             callback(
                                 {
@@ -261,45 +255,6 @@ def start_monitor(
                     logger.error("Error when processing event %s", exc_info=e)
 
                 time.sleep(polling_interval)
-    except Exception:
-        logger.debug(traceback.format_exc())
-
 
-def update_everserver_status(
-    everserver_status_path: str, status: ExperimentState, message: str | None = None
-) -> None:
-    """Update the everest server status with new status information"""
-    new_status = {"status": status, "message": message}
-    path = everserver_status_path
-    if not os.path.exists(os.path.dirname(path)):
-        os.makedirs(os.path.dirname(path))
-        with open(path, "w", encoding="utf-8") as outfile:
-            json.dump(new_status, outfile)
-    elif os.path.exists(path):
-        server_status = everserver_status(path)
-        if server_status["message"] is not None:
-            if message is not None:
-                new_status["message"] = "{}\n{}".format(
-                    server_status["message"], message
-                )
-            else:
-                new_status["message"] = server_status["message"]
-        with open(path, "w", encoding="utf-8") as outfile:
-            json.dump(new_status, outfile)
-
-
-def everserver_status(everserver_status_path: str) -> dict[str, Any]:
-    """Returns a dictionary representing the everest server status. If the
-    status file is not found we assume the server has never ran before, and will
-    return a status of ServerStatus.never_run
-
-    Example: {
-                'status': ServerStatus.completed
-                'message': None
-             }
-    """
-    if os.path.exists(everserver_status_path):
-        with open(everserver_status_path, encoding="utf-8") as f:
-            return json.load(f)
-    else:
-        return {"status": ExperimentState.never_run, "message": None}
+    except Exception:
+        logger.exception(traceback.format_exc())

everest/detached/everserver.py

@@ -4,7 +4,6 @@ import logging.config
 import os
 import pathlib
 import time
-import traceback
 from pathlib import Path
 from tempfile import NamedTemporaryFile
 from typing import Any
@@ -13,20 +12,17 @@ import yaml
 from opentelemetry.trace.propagation.tracecontext import TraceContextTextMapPropagator
 
 from ert.plugins.plugin_manager import ErtPluginManager
-from ert.run_models.run_model import ExperimentStatus
 from ert.services import StorageService
 from ert.services._base_service import BaseServiceExit
+from ert.storage import ExperimentStatus
+from ert.storage.local_experiment import ExperimentState
 from ert.trace import tracer
 from everest.config import ServerConfig
-from everest.detached import (
-    ExperimentState,
-    everserver_status,
-    update_everserver_status,
-)
 from everest.strings import (
     DEFAULT_LOGGING_FORMAT,
     EVEREST,
     EVERSERVER,
+    EXPERIMENT_SERVER,
     OPTIMIZATION_LOG_DIR,
 )
 from everest.util import makedirs_if_needed, version_info
@@ -67,6 +63,11 @@ def _configure_loggers(
                 "level": logging_level,
                 "propagate": False,
             },
+            EXPERIMENT_SERVER: {
+                "handlers": ["endpoint_log"],
+                "level": logging_level,
+                "propagate": False,
+            },
             EVEREST: {
                 "handlers": ["everest_log"],
                 "level": logging_level,
@@ -93,8 +94,9 @@ def _configure_loggers(
 
     yaml.add_representer(pathlib.PosixPath, path_representer)
     if output_file:
-        with open(output_file, "w", encoding="utf-8") as outfile:
-            yaml.dump(logging_config, outfile, default_flow_style=False)
+        Path(output_file).write_text(
+            yaml.dump(logging_config, default_flow_style=False), encoding="utf-8"
+        )
     logging.config.dictConfig(logging_config)
 
     plugin_manager = ErtPluginManager()
@@ -135,8 +137,6 @@ def main() -> None:
 
     output_dir = options.output_dir
 
-    status_path = ServerConfig.get_everserver_status_path(output_dir)
-
     ctx = (
         TraceContextTextMapPropagator().extract(
             carrier={"traceparent": options.traceparent}
@@ -158,7 +158,6 @@ def main() -> None:
             )
 
             logging.getLogger(EVERSERVER).info("Everserver starting ...")
-            update_everserver_status(status_path, ExperimentState.pending)
             logger.info(version_info())
             logger.info(f"Output directory: {output_dir}")
             # Starting the server
@@ -168,8 +167,7 @@ def main() -> None:
                 timeout=240, project=server_path, logging_config=log_file.name
             ) as server:
                 server.fetch_conn_info()
-                with StorageService.session(project=server_path) as client:
-                    update_everserver_status(status_path, ExperimentState.running)
+                with StorageService.session(project=Path(server_path)) as client:
                     done = False
                     while not done:
                         response = client.get(
@@ -181,33 +179,10 @@ def main() -> None:
                             ExperimentState.running,
                         }
                         time.sleep(0.5)
-                    if status.status == ExperimentState.completed:
-                        update_everserver_status(
-                            status_path,
-                            ExperimentState.completed,
-                            message=status.message,
-                        )
-                    elif status.status == ExperimentState.stopped:
-                        update_everserver_status(
-                            status_path,
-                            ExperimentState.stopped,
-                            message=status.message,
-                        )
-                    elif status.status == ExperimentState.failed:
-                        update_everserver_status(
-                            status_path, ExperimentState.failed, message=status.message
-                        )
         except BaseServiceExit:
             # Server exit, happens on normal shutdown and keyboard interrupt
-            server_status = everserver_status(status_path)
-            if server_status["status"] == ExperimentState.running:
-                update_everserver_status(status_path, ExperimentState.stopped)
+            logging.getLogger(EVERSERVER).info("Everserver stopped by user")
         except Exception as e:
-            update_everserver_status(
-                status_path,
-                ExperimentState.failed,
-                message=traceback.format_exc(),
-            )
             logging.getLogger(EVERSERVER).exception(e)
 
 

everest/everest_storage.py

@@ -12,7 +12,7 @@ import numpy as np
 import polars as pl
 from ropt.results import FunctionResults, GradientResults, Results
 
-from everest.config.objective_function_config import ObjectiveFunctionConfig
+from ert.config import EverestObjectivesConfig
 from everest.config.output_constraint_config import OutputConstraintConfig
 from everest.strings import EVEREST
 
@@ -153,16 +153,12 @@ class BatchStorageData:
 
     @cached_property
     def is_improvement(self) -> bool:
-        with open(self._path / "batch.json", encoding="utf-8") as f:
-            info = json.load(f)
-
+        info = json.loads((self._path / "batch.json").read_text(encoding="utf-8"))
         return bool(info["is_improvement"])
 
     @cached_property
     def batch_id(self) -> bool:
-        with open(self._path / "batch.json", encoding="utf-8") as f:
-            info = json.load(f)
-
+        info = json.loads((self._path / "batch.json").read_text(encoding="utf-8"))
         return info["batch_id"]
 
     def write_metadata(self, is_improvement: bool) -> None:
@@ -170,15 +166,9 @@ class BatchStorageData:
         if "is_improvement" in self.__dict__:
             del self.is_improvement
 
-        with open(self._path / "batch.json", encoding="utf-8") as f:
-            info = json.load(f)
-            info["is_improvement"] = is_improvement
-
-        with open(self._path / "batch.json", "w", encoding="utf-8") as f:
-            json.dump(
-                info,
-                f,
-            )
+        info = json.loads((self._path / "batch.json").read_text(encoding="utf-8"))
+        info["is_improvement"] = is_improvement
+        (self._path / "batch.json").write_text(json.dumps(info), encoding="utf-8")
 
 
 class FunctionBatchStorageData(BatchStorageData):
@@ -541,7 +531,7 @@ class EverestStorage:
     def init(
         self,
         formatted_control_names: list[str],
-        objective_functions: list[ObjectiveFunctionConfig],
+        objective_functions: EverestObjectivesConfig,
         output_constraints: list[OutputConstraintConfig] | None,
         realizations: list[int],
     ) -> None:
@@ -554,18 +544,21 @@ class EverestStorage:
         # TODO: The weight and normalization keys are only used by the everest api,
         # with everviz. They should be removed in the long run.
         weights = np.fromiter(
-            (1.0 if obj.weight is None else obj.weight for obj in objective_functions),
+            (
+                1.0 if weight is None else weight
+                for weight in objective_functions.weights
+            ),
             dtype=np.float64,
         )
 
         objective_functions_dataframe = pl.DataFrame(
             {
-                "objective_name": [objective.name for objective in objective_functions],
+                "objective_name": objective_functions.keys,
                 "weight": pl.Series(weights / sum(weights), dtype=pl.Float64),
                 "scale": pl.Series(
                     [
-                        1.0 if obj.scale is None else obj.scale
-                        for obj in objective_functions
+                        1.0 if scale is None else scale
+                        for scale in objective_functions.scales
                     ],
                     dtype=pl.Float64,
                 ),
@@ -906,18 +899,15 @@ class EverestStorage:
         for batch_id, batch_dict in batch_dicts.items():
             target_ensemble = self._experiment.get_ensemble_by_name(f"batch_{batch_id}")
 
-            with open(
-                target_ensemble.optimizer_mount_point / "batch.json",
-                "w+",
-                encoding="utf-8",
-            ) as f:
-                json.dump(
+            (target_ensemble.optimizer_mount_point / "batch.json").write_text(
+                json.dumps(
                     {
                         "batch_id": batch_id,
                         "is_improvement": False,
                     },
-                    f,
-                )
+                ),
+                encoding="utf-8",
+            )
 
             batch_data = BatchStorageData(path=target_ensemble.optimizer_mount_point)
 

everest/optimizer/everest2ropt.py

@@ -3,11 +3,11 @@ from typing import Any
 
 from ropt.enums import PerturbationType, VariableType
 
+from ert.config import EverestObjectivesConfig
 from everest.config import (
     ControlConfig,
     InputConstraintConfig,
     ModelConfig,
-    ObjectiveFunctionConfig,
     OptimizationConfig,
     OutputConstraintConfig,
 )
@@ -48,16 +48,15 @@ def _parse_controls(
 
 
 def _parse_objectives(
-    objective_functions: list[ObjectiveFunctionConfig],
+    objective_functions: EverestObjectivesConfig,
 ) -> tuple[dict[str, Any], list[dict[str, Any]]]:
-    weights: list[float] = []
+    weights: list[float] = [
+        1.0 if weight is None else weight for weight in objective_functions.weights
+    ]
     function_estimator_indices: list[int] = []
     function_estimators: list = []  # type: ignore
 
-    for objective in objective_functions:
-        assert isinstance(objective.name, str)
-        weights.append(objective.weight or 1.0)
-
+    for objective_type in objective_functions.objective_types:
         # If any objective specifies an objective type, we have to specify
         # function estimators in ropt to implement these types. This is done by
         # supplying a list of estimators and for each objective an index into
@@ -66,14 +65,14 @@ def _parse_objectives(
             (
                 idx
                 for idx, estimator in enumerate(function_estimators)
-                if estimator["method"] == objective.type
+                if estimator["method"] == objective_type
             ),
             None,
         )
         # If not, make a new estimator:
         if function_estimator_idx is None:
             function_estimator_idx = len(function_estimators)
-            function_estimators.append({"method": objective.type})
+            function_estimators.append({"method": objective_type})
         function_estimator_indices.append(function_estimator_idx)
 
     ropt_objectives: dict[str, Any] = {"weights": weights}
@@ -250,7 +249,7 @@ def _parse_optimization(
 
 def everest2ropt(
     controls: list[ControlConfig],
-    objective_functions: list[ObjectiveFunctionConfig],
+    objective_functions: EverestObjectivesConfig,
     input_constraints: list[InputConstraintConfig],
     output_constraints: list[OutputConstraintConfig],
     optimization: OptimizationConfig | None,
@@ -297,7 +296,7 @@ def everest2ropt(
             "variable": [
                 name for config in controls for name in config.formatted_control_names
             ],
-            "objective": [objective.name for objective in objective_functions],
+            "objective": objective_functions.keys,
             "nonlinear_constraint": [
                 constraint.name for constraint in output_constraints
             ],

everest/optimizer/opt_model_transforms.py

@@ -11,11 +11,11 @@ from ropt.transforms.base import (
     VariableTransform,
 )
 
+from ert.config import EverestObjectivesConfig
 from everest.config import (
     ControlConfig,
     InputConstraintConfig,
     ModelConfig,
-    ObjectiveFunctionConfig,
     OutputConstraintConfig,
 )
 from everest.config.utils import FlattenedControls
@@ -430,7 +430,7 @@ class ConstraintScaler(NonLinearConstraintTransform):
 
 def get_optimization_domain_transforms(
     controls: list[ControlConfig],
-    objectives: list[ObjectiveFunctionConfig],
+    objectives: EverestObjectivesConfig,
     input_constraints: list[InputConstraintConfig] | None,
     output_constraints: list[OutputConstraintConfig] | None,
     model: ModelConfig,
@@ -461,14 +461,10 @@ def get_optimization_domain_transforms(
 
     objective_scaler = ObjectiveScaler(
         auto_scale=auto_scale,
-        scales=[
-            1.0 if objective.scale is None else objective.scale
-            for objective in objectives
-        ],
+        scales=[1.0 if scale is None else scale for scale in objectives.scales],
         realization_weights=realization_weights,
         objective_weights=[
-            1.0 if objective.weight is None else objective.weight
-            for objective in objectives
+            1.0 if weight is None else weight for weight in objectives.weights
         ],
     )