ert 16.0.9__py3-none-any.whl → 19.0.0rc2__py3-none-any.whl

everest/detached/client.py

@@ -1,7 +1,5 @@
 import asyncio
-import json
 import logging
-import os
 import re
 import ssl
 import time
@@ -17,9 +15,7 @@ from websockets import ConnectionClosedError, ConnectionClosedOK
 from websockets.sync.client import connect
 
 from ert.dark_storage.client import Client
-from ert.ensemble_evaluator import EndEvent
 from ert.run_models.event import EverestBatchResultEvent, status_event_from_json
-from ert.run_models.run_model import ExperimentState
 from ert.scheduler import create_driver
 from ert.scheduler.driver import Driver, FailedSubmit
 from ert.scheduler.event import StartedEvent
@@ -49,7 +45,7 @@ async def start_server(config: EverestConfig, logging_level: int) -> Driver:
     """
     Start an Everest server running the optimization defined in the config
     """
-    driver = create_driver(config.server.queue_system)  # type: ignore
+    driver = create_driver(config.server.queue_system, poll_period=0.1)  # type: ignore
     try:
         args = [
             "--output-dir",
@@ -85,7 +81,7 @@ def stop_server(
     for retry in range(retries):
         try:
             url, cert, auth = server_context
-            stop_endpoint = "/".join([url, EverEndpoints.stop])
+            stop_endpoint = f"{url}/{EverEndpoints.stop}"
             response = requests.post(
                 stop_endpoint,
                 verify=cert,
@@ -109,7 +105,7 @@ def start_experiment(
     for retry in range(retries):
         try:
             url, cert, auth = server_context
-            start_endpoint = "/".join([url, EverEndpoints.start_experiment])
+            start_endpoint = f"{url}/{EverEndpoints.start_experiment}"
             response = requests.post(
                 start_endpoint,
                 verify=cert,
@@ -130,7 +126,7 @@ def extract_errors_from_file(path: str) -> list[str]:
     return re.findall(r"(Error \w+.*)", Path(path).read_text(encoding="utf-8"))
 
 
-def wait_for_server(client: Client, timeout: int | float) -> None:
+def wait_for_server(client: Client, timeout: float) -> None:
     """
     Waits until the everest server has started. Polls every second
     for server availability until timeout (measured in seconds).
@@ -236,9 +232,7 @@ def start_monitor(
                 try:
                     message = websocket.recv(timeout=1.0)
                     event = status_event_from_json(message)
-                    if isinstance(event, EndEvent):
-                        print(event.msg)
-                    elif isinstance(event, EverestBatchResultEvent):
+                    if isinstance(event, EverestBatchResultEvent):
                         if event.result_type == "FunctionResult":
                             callback(
                                 {
@@ -261,45 +255,6 @@ def start_monitor(
                     logger.error("Error when processing event %s", exc_info=e)
 
                 time.sleep(polling_interval)
-    except Exception:
-        logger.debug(traceback.format_exc())
-
 
-def update_everserver_status(
-    everserver_status_path: str, status: ExperimentState, message: str | None = None
-) -> None:
-    """Update the everest server status with new status information"""
-    new_status = {"status": status, "message": message}
-    path = everserver_status_path
-    if not os.path.exists(os.path.dirname(path)):
-        os.makedirs(os.path.dirname(path))
-        with open(path, "w", encoding="utf-8") as outfile:
-            json.dump(new_status, outfile)
-    elif os.path.exists(path):
-        server_status = everserver_status(path)
-        if server_status["message"] is not None:
-            if message is not None:
-                new_status["message"] = "{}\n{}".format(
-                    server_status["message"], message
-                )
-            else:
-                new_status["message"] = server_status["message"]
-        with open(path, "w", encoding="utf-8") as outfile:
-            json.dump(new_status, outfile)
-
-
-def everserver_status(everserver_status_path: str) -> dict[str, Any]:
-    """Returns a dictionary representing the everest server status. If the
-    status file is not found we assume the server has never ran before, and will
-    return a status of ServerStatus.never_run
-
-    Example: {
-                'status': ServerStatus.completed
-                'message': None
-             }
-    """
-    if os.path.exists(everserver_status_path):
-        with open(everserver_status_path, encoding="utf-8") as f:
-            return json.load(f)
-    else:
-        return {"status": ExperimentState.never_run, "message": None}
+    except Exception:
+        logger.exception(traceback.format_exc())

everest/detached/everserver.py

@@ -4,7 +4,6 @@ import logging.config
 import os
 import pathlib
 import time
-import traceback
 from pathlib import Path
 from tempfile import NamedTemporaryFile
 from typing import Any
@@ -13,20 +12,17 @@ import yaml
 from opentelemetry.trace.propagation.tracecontext import TraceContextTextMapPropagator
 
 from ert.plugins.plugin_manager import ErtPluginManager
-from ert.run_models.run_model import ExperimentStatus
-from ert.services import StorageService
+from ert.services import ErtServer
 from ert.services._base_service import BaseServiceExit
+from ert.storage import ExperimentStatus
+from ert.storage.local_experiment import ExperimentState
 from ert.trace import tracer
 from everest.config import ServerConfig
-from everest.detached import (
-    ExperimentState,
-    everserver_status,
-    update_everserver_status,
-)
 from everest.strings import (
     DEFAULT_LOGGING_FORMAT,
     EVEREST,
     EVERSERVER,
+    EXPERIMENT_SERVER,
     OPTIMIZATION_LOG_DIR,
 )
 from everest.util import makedirs_if_needed, version_info
@@ -35,7 +31,7 @@ logger = logging.getLogger(__name__)
 
 
 def _configure_loggers(
-    detached_dir: Path, log_dir: Path, logging_level: int, output_file: str | None
+    log_dir: Path, logging_level: int, output_file: str | None
 ) -> None:
     def make_handler_config(path: Path, log_level: int) -> dict[str, Any]:
         makedirs_if_needed(str(path.parent))
@@ -50,7 +46,7 @@ def _configure_loggers(
         "version": 1,
         "handlers": {
             "endpoint_log": make_handler_config(
-                detached_dir / "endpoint.log", logging_level
+                log_dir / "everserver.log", logging_level
             ),
             "everest_log": make_handler_config(log_dir / "everest.log", logging_level),
             "forward_models_log": make_handler_config(
@@ -67,6 +63,11 @@ def _configure_loggers(
                 "level": logging_level,
                 "propagate": False,
             },
+            EXPERIMENT_SERVER: {
+                "handlers": ["endpoint_log"],
+                "level": logging_level,
+                "propagate": False,
+            },
             EVEREST: {
                 "handlers": ["everest_log"],
                 "level": logging_level,
@@ -93,8 +94,9 @@ def _configure_loggers(
 
     yaml.add_representer(pathlib.PosixPath, path_representer)
     if output_file:
-        with open(output_file, "w", encoding="utf-8") as outfile:
-            yaml.dump(logging_config, outfile, default_flow_style=False)
+        Path(output_file).write_text(
+            yaml.dump(logging_config, default_flow_style=False), encoding="utf-8"
+        )
     logging.config.dictConfig(logging_config)
 
     plugin_manager = ErtPluginManager()
@@ -135,8 +137,6 @@ def main() -> None:
 
     output_dir = options.output_dir
 
-    status_path = ServerConfig.get_everserver_status_path(output_dir)
-
     ctx = (
         TraceContextTextMapPropagator().extract(
             carrier={"traceparent": options.traceparent}
@@ -151,25 +151,22 @@ def main() -> None:
     ):
         try:
             _configure_loggers(
-                detached_dir=Path(ServerConfig.get_detached_node_dir(output_dir)),
                 log_dir=Path(output_dir) / OPTIMIZATION_LOG_DIR,
                 logging_level=options.logging_level,
                 output_file=log_file.name,
             )
 
             logging.getLogger(EVERSERVER).info("Everserver starting ...")
-            update_everserver_status(status_path, ExperimentState.pending)
             logger.info(version_info())
             logger.info(f"Output directory: {output_dir}")
             # Starting the server
             server_path = os.path.abspath(ServerConfig.get_session_dir(output_dir))
             status = ""
-            with StorageService.init_service(
-                timeout=240, project=server_path, logging_config=log_file.name
+            with ErtServer.init_service(
+                timeout=240, project=Path(server_path), logging_config=log_file.name
             ) as server:
-                server.fetch_conn_info()
-                with StorageService.session(project=server_path) as client:
-                    update_everserver_status(status_path, ExperimentState.running)
+                server.fetch_connection_info()
+                with ErtServer.session(project=Path(server_path)) as client:
                     done = False
                     while not done:
                         response = client.get(
@@ -181,33 +178,10 @@ def main() -> None:
                             ExperimentState.running,
                         }
                         time.sleep(0.5)
-                    if status.status == ExperimentState.completed:
-                        update_everserver_status(
-                            status_path,
-                            ExperimentState.completed,
-                            message=status.message,
-                        )
-                    elif status.status == ExperimentState.stopped:
-                        update_everserver_status(
-                            status_path,
-                            ExperimentState.stopped,
-                            message=status.message,
-                        )
-                    elif status.status == ExperimentState.failed:
-                        update_everserver_status(
-                            status_path, ExperimentState.failed, message=status.message
-                        )
         except BaseServiceExit:
             # Server exit, happens on normal shutdown and keyboard interrupt
-            server_status = everserver_status(status_path)
-            if server_status["status"] == ExperimentState.running:
-                update_everserver_status(status_path, ExperimentState.stopped)
+            logging.getLogger(EVERSERVER).info("Everserver stopped by user")
         except Exception as e:
-            update_everserver_status(
-                status_path,
-                ExperimentState.failed,
-                message=traceback.format_exc(),
-            )
             logging.getLogger(EVERSERVER).exception(e)
 
 

everest/everest_storage.py

@@ -12,8 +12,7 @@ import numpy as np
 import polars as pl
 from ropt.results import FunctionResults, GradientResults, Results
 
-from everest.config.objective_function_config import ObjectiveFunctionConfig
-from everest.config.output_constraint_config import OutputConstraintConfig
+from ert.config import EverestConstraintsConfig, EverestObjectivesConfig
 from everest.strings import EVEREST
 
 logger = logging.getLogger(__name__)
@@ -153,16 +152,12 @@ class BatchStorageData:
 
     @cached_property
     def is_improvement(self) -> bool:
-        with open(self._path / "batch.json", encoding="utf-8") as f:
-            info = json.load(f)
-
+        info = json.loads((self._path / "batch.json").read_text(encoding="utf-8"))
         return bool(info["is_improvement"])
 
     @cached_property
     def batch_id(self) -> bool:
-        with open(self._path / "batch.json", encoding="utf-8") as f:
-            info = json.load(f)
-
+        info = json.loads((self._path / "batch.json").read_text(encoding="utf-8"))
         return info["batch_id"]
 
     def write_metadata(self, is_improvement: bool) -> None:
@@ -170,15 +165,9 @@ class BatchStorageData:
         if "is_improvement" in self.__dict__:
             del self.is_improvement
 
-        with open(self._path / "batch.json", encoding="utf-8") as f:
-            info = json.load(f)
-            info["is_improvement"] = is_improvement
-
-        with open(self._path / "batch.json", "w", encoding="utf-8") as f:
-            json.dump(
-                info,
-                f,
-            )
+        info = json.loads((self._path / "batch.json").read_text(encoding="utf-8"))
+        info["is_improvement"] = is_improvement
+        (self._path / "batch.json").write_text(json.dumps(info), encoding="utf-8")
 
 
 class FunctionBatchStorageData(BatchStorageData):
@@ -541,8 +530,8 @@ class EverestStorage:
     def init(
         self,
         formatted_control_names: list[str],
-        objective_functions: list[ObjectiveFunctionConfig],
-        output_constraints: list[OutputConstraintConfig] | None,
+        objective_functions: EverestObjectivesConfig,
+        output_constraints: EverestConstraintsConfig | None,
         realizations: list[int],
     ) -> None:
         controls = pl.DataFrame(
@@ -551,21 +540,22 @@ class EverestStorage:
             }
         )
 
-        # TODO: The weight and normalization keys are only used by the everest api,
-        # with everviz. They should be removed in the long run.
         weights = np.fromiter(
-            (1.0 if obj.weight is None else obj.weight for obj in objective_functions),
+            (
+                1.0 if weight is None else weight
+                for weight in objective_functions.weights
+            ),
             dtype=np.float64,
         )
 
         objective_functions_dataframe = pl.DataFrame(
             {
-                "objective_name": [objective.name for objective in objective_functions],
+                "objective_name": objective_functions.keys,
                 "weight": pl.Series(weights / sum(weights), dtype=pl.Float64),
                 "scale": pl.Series(
                     [
-                        1.0 if obj.scale is None else obj.scale
-                        for obj in objective_functions
+                        1.0 if scale is None else scale
+                        for scale in objective_functions.scales
                     ],
                     dtype=pl.Float64,
                 ),
@@ -573,13 +563,7 @@ class EverestStorage:
         )
 
         nonlinear_constraints = (
-            pl.DataFrame(
-                {
-                    "constraint_name": [
-                        constraint.name for constraint in output_constraints
-                    ],
-                }
-            )
+            pl.DataFrame({"constraint_name": output_constraints.keys})
             if output_constraints
             else None
         )
@@ -906,18 +890,15 @@ class EverestStorage:
         for batch_id, batch_dict in batch_dicts.items():
             target_ensemble = self._experiment.get_ensemble_by_name(f"batch_{batch_id}")
 
-            with open(
-                target_ensemble.optimizer_mount_point / "batch.json",
-                "w+",
-                encoding="utf-8",
-            ) as f:
-                json.dump(
+            (target_ensemble.optimizer_mount_point / "batch.json").write_text(
+                json.dumps(
                     {
                         "batch_id": batch_id,
                         "is_improvement": False,
                     },
-                    f,
-                )
+                ),
+                encoding="utf-8",
+            )
 
             batch_data = BatchStorageData(path=target_ensemble.optimizer_mount_point)
 
@@ -1011,6 +992,9 @@ class EverestStorage:
     def export_dataframes(
         self,
     ) -> tuple[pl.DataFrame, pl.DataFrame, pl.DataFrame]:
+        if not self.data.batches:
+            return (pl.DataFrame(), pl.DataFrame(), pl.DataFrame())
+
         batch_dfs_to_join = {}  # type: ignore
         realization_dfs_to_join = {}  # type: ignore
         perturbation_dfs_to_join = {}  # type: ignore

everest/gui/everest_client.py

@@ -44,7 +44,7 @@ class EverestClient:
 
     def _http_get(self, endpoint: EverEndpoints) -> requests.Response:
         return requests.get(
-            "/".join([self._url, endpoint]),
+            f"{self._url}/{endpoint}",
             verify=self._cert,
             auth=(self._username, self._password),
             proxies={"http": None, "https": None},  # type: ignore
@@ -52,7 +52,7 @@ class EverestClient:
 
     def _http_post(self, endpoint: EverEndpoints) -> requests.Response:
         return requests.post(
-            "/".join([self._url, endpoint]),
+            f"{self._url}/{endpoint}",
             verify=self._cert,
             auth=(self._username, self._password),
             proxies={"http": None, "https": None},  # type: ignore
@@ -128,7 +128,6 @@ class EverestClient:
             supports_rerunning_failed_realizations=False,
             start_simulations_thread=start_fn,
             cancel=self.stop,
-            get_runtime=self.get_runtime,
             has_failed_realizations=lambda: False,
         )
 

everest/gui/main_window.py

@@ -13,7 +13,7 @@ from PyQt6.QtWidgets import (
 from ert.gui.ertnotifier import ErtNotifier
 from ert.gui.simulation.run_dialog import RunDialog
 from ert.plugins import ErtPluginManager
-from ert.services import StorageService
+from ert.services import ErtServer
 from everest.config import ServerConfig
 from everest.detached import wait_for_server
 from everest.gui.everest_client import EverestClient
@@ -43,7 +43,7 @@ class EverestMainWindow(QMainWindow):
         self.setCentralWidget(self.central_widget)
 
     def run(self) -> None:
-        storage_client = StorageService.session(
+        storage_client = ErtServer.session(
             Path(ServerConfig.get_session_dir(self.output_dir))
         )
         wait_for_server(storage_client, 60)

everest/optimizer/everest2ropt.py

@@ -3,34 +3,48 @@ from typing import Any
 
 from ropt.enums import PerturbationType, VariableType
 
+from ert.config import EverestConstraintsConfig, EverestControl, EverestObjectivesConfig
 from everest.config import (
-    ControlConfig,
     InputConstraintConfig,
     ModelConfig,
-    ObjectiveFunctionConfig,
     OptimizationConfig,
     OutputConstraintConfig,
 )
-from everest.config.utils import FlattenedControls
+from everest.config.utils import get_samplers
 
 
 def _parse_controls(
-    controls: FlattenedControls, random_seed: int
+    controls: list[EverestControl], random_seed: int
 ) -> tuple[dict[str, Any], list[dict[str, Any]] | None]:
-    control_types = [VariableType[type_.upper()] for type_ in controls.types]
+    control_types = [
+        VariableType[type_.upper()]
+        for control in controls
+        for type_ in control.control_types
+    ]
+    initial_guesses = [
+        initial_guess
+        for control in controls
+        for initial_guess in control.initial_guesses
+    ]
+    samplers, sampler_indices = get_samplers(controls)
     ropt_variables: dict[str, Any] = {
         "types": None if all(item is None for item in control_types) else control_types,
-        "variable_count": len(controls.initial_guesses),
-        "lower_bounds": controls.lower_bounds,
-        "upper_bounds": controls.upper_bounds,
+        "variable_count": len(initial_guesses),
+        "lower_bounds": [min_ for control in controls for min_ in control.min],
+        "upper_bounds": [max_ for control in controls for max_ in control.max],
         "perturbation_types": [
             PerturbationType[perturbation_type.upper()]
-            for perturbation_type in controls.perturbation_types
+            for control in controls
+            for perturbation_type in control.perturbation_types
         ],
-        "perturbation_magnitudes": controls.perturbation_magnitudes,
-        "mask": controls.enabled,
+        "perturbation_magnitudes": [
+            perturbation_magnitude
+            for control in controls
+            for perturbation_magnitude in control.perturbation_magnitudes
+        ],
+        "mask": [enabled for control in controls for enabled in control.enabled],
         "seed": random_seed,
-        "samplers": controls.sampler_indices,
+        "samplers": sampler_indices,
     }
 
     ropt_samplers = [
@@ -41,23 +55,22 @@ def _parse_controls(
             "options": {} if sampler.options is None else sampler.options,
             "shared": False if sampler.shared is None else sampler.shared,
         }
-        for sampler in controls.samplers
+        for sampler in samplers
     ]
 
     return ropt_variables, ropt_samplers
 
 
 def _parse_objectives(
-    objective_functions: list[ObjectiveFunctionConfig],
+    objective_functions: EverestObjectivesConfig,
 ) -> tuple[dict[str, Any], list[dict[str, Any]]]:
-    weights: list[float] = []
+    weights: list[float] = [
+        1.0 if weight is None else weight for weight in objective_functions.weights
+    ]
     function_estimator_indices: list[int] = []
     function_estimators: list = []  # type: ignore
 
-    for objective in objective_functions:
-        assert isinstance(objective.name, str)
-        weights.append(objective.weight or 1.0)
-
+    for objective_type in objective_functions.objective_types:
         # If any objective specifies an objective type, we have to specify
         # function estimators in ropt to implement these types. This is done by
         # supplying a list of estimators and for each objective an index into
@@ -66,14 +79,14 @@ def _parse_objectives(
             (
                 idx
                 for idx, estimator in enumerate(function_estimators)
-                if estimator["method"] == objective.type
+                if estimator["method"] == objective_type
             ),
             None,
         )
         # If not, make a new estimator:
         if function_estimator_idx is None:
             function_estimator_idx = len(function_estimators)
-            function_estimators.append({"method": objective.type})
+            function_estimators.append({"method": objective_type})
         function_estimator_indices.append(function_estimator_idx)
 
     ropt_objectives: dict[str, Any] = {"weights": weights}
@@ -103,13 +116,13 @@ def _get_bounds(
 
 def _parse_input_constraints(
     input_constraints: list[InputConstraintConfig],
-    controls: list[ControlConfig],
+    controls: list[EverestControl],
 ) -> dict[str, Any]:
     formatted_control_names = [
-        name for config in controls for name in config.formatted_control_names
+        name for config in controls for name in config.input_keys
     ]
     formatted_control_names_dotdash = [
-        name for config in controls for name in config.formatted_control_names_dotdash
+        name for config in controls for name in config.input_keys_dotdash
     ]
 
     def _get_control_index(name: str) -> int:
@@ -144,13 +157,26 @@ def _parse_input_constraints(
 
 
 def _parse_output_constraints(
-    output_constraints: list[OutputConstraintConfig],
+    output_constraints: EverestConstraintsConfig | None,
 ) -> dict[str, Any]:
     if output_constraints:
-        lower_bounds, upper_bounds = _get_bounds(output_constraints)
         return {
-            "lower_bounds": lower_bounds,
-            "upper_bounds": upper_bounds,
+            "lower_bounds": [
+                target if target is not None else lb
+                for lb, target in zip(
+                    output_constraints.lower_bounds,
+                    output_constraints.targets,
+                    strict=False,
+                )
+            ],
+            "upper_bounds": [
+                target if target is not None else ub
+                for ub, target in zip(
+                    output_constraints.upper_bounds,
+                    output_constraints.targets,
+                    strict=False,
+                )
+            ],
         }
     return {}
 
@@ -249,18 +275,16 @@ def _parse_optimization(
 
 
 def everest2ropt(
-    controls: list[ControlConfig],
-    objective_functions: list[ObjectiveFunctionConfig],
+    controls: list[EverestControl],
+    objective_functions: EverestObjectivesConfig,
     input_constraints: list[InputConstraintConfig],
-    output_constraints: list[OutputConstraintConfig],
+    output_constraints: EverestConstraintsConfig | None,
     optimization: OptimizationConfig | None,
     model: ModelConfig,
     random_seed: int,
     optimization_output_dir: str,
 ) -> tuple[dict[str, Any], list[float]]:
-    flattened_controls = FlattenedControls(controls)
-
-    ropt_variables, ropt_samplers = _parse_controls(flattened_controls, random_seed)
+    ropt_variables, ropt_samplers = _parse_controls(controls, random_seed)
     ropt_objectives, ropt_function_estimators = _parse_objectives(objective_functions)
     ropt_linear_constraints = _parse_input_constraints(input_constraints, controls)
     ropt_nonlinear_constraints = _parse_output_constraints(output_constraints)
@@ -294,13 +318,11 @@ def everest2ropt(
         "realizations": ropt_realizations,
         "optimizer": ropt_optimizer,
         "names": {
-            "variable": [
-                name for config in controls for name in config.formatted_control_names
-            ],
-            "objective": [objective.name for objective in objective_functions],
-            "nonlinear_constraint": [
-                constraint.name for constraint in output_constraints
-            ],
+            "variable": [name for config in controls for name in config.input_keys],
+            "objective": objective_functions.keys,
+            "nonlinear_constraint": output_constraints.keys
+            if output_constraints is not None
+            else [],
             "realization": model.realizations,
         },
     }
@@ -317,4 +339,8 @@ def everest2ropt(
     if ropt_samplers:
         ropt_config["samplers"] = ropt_samplers
 
-    return ropt_config, flattened_controls.initial_guesses
+    return ropt_config, [
+        initial_guess
+        for control in controls
+        for initial_guess in control.initial_guesses
+    ]