ert 16.0.9__py3-none-any.whl → 19.0.0rc2__py3-none-any.whl

ert/dark_storage/endpoints/experiment_server.py

@@ -9,6 +9,7 @@ import traceback
 import uuid
 from base64 import b64decode
 from queue import SimpleQueue
+from typing import Annotated
 
 from fastapi import (
     APIRouter,
@@ -25,11 +26,12 @@ from starlette.requests import Request
 from starlette.responses import PlainTextResponse, Response
 from starlette.websockets import WebSocket
 
+from ert.base_model_context import use_runtime_plugins
 from ert.config import QueueSystem
 from ert.ensemble_evaluator import EndEvent, EvaluatorServerConfig
 from ert.ensemble_evaluator.event import FullSnapshotEvent, SnapshotUpdateEvent
 from ert.ensemble_evaluator.snapshot import EnsembleSnapshot
-from ert.plugins import ErtPluginContext
+from ert.plugins import get_site_plugins
 from ert.run_models import StatusEvents
 from ert.run_models.everest_run_model import EverestExitCode, EverestRunModel
 from everest.config import EverestConfig
@@ -38,7 +40,7 @@ from everest.detached.everserver import (
     ExperimentStatus,
 )
 from everest.strings import (
-    EVERSERVER,
+    EXPERIMENT_SERVER,
     OPT_FAILURE_ALL_REALIZATIONS,
     OPT_FAILURE_REALIZATIONS,
     EverEndpoints,
@@ -115,7 +117,7 @@ def _get_optimization_status(
             status_ = ExperimentState.failed
             messages = _failed_realizations_messages(events, exit_code)
             for msg in messages:
-                logging.getLogger(EVERSERVER).error(msg)
+                logging.getLogger(EXPERIMENT_SERVER).error(msg)
             return status_, "\n".join(messages)
         case _:
             return ExperimentState.completed, "Optimization completed."
@@ -143,7 +145,7 @@ def _check_user(credentials: HTTPBasicCredentials) -> None:
 
 
 def _log(request: Request) -> None:
-    logging.getLogger(EVERSERVER).debug(
+    logging.getLogger(EXPERIMENT_SERVER).debug(
         f"{request.scope['path']} entered from "
         f"{request.client.host if request.client else 'unknown host'} "
         f"with HTTP {request.method}"
@@ -152,7 +154,7 @@ def _log(request: Request) -> None:
 
 @router.get("/")
 def get_status(
-    request: Request, credentials: HTTPBasicCredentials = Depends(security)
+    request: Request, credentials: Annotated[HTTPBasicCredentials, Depends(security)]
 ) -> PlainTextResponse:
     _log(request)
     _check_user(credentials)
@@ -161,7 +163,7 @@ def get_status(
 
 @router.get("/status")
 def experiment_status(
-    request: Request, credentials: HTTPBasicCredentials = Depends(security)
+    request: Request, credentials: Annotated[HTTPBasicCredentials, Depends(security)]
 ) -> ExperimentStatus:
     _log(request)
     _check_user(credentials)
@@ -170,7 +172,7 @@ def experiment_status(
 
 @router.post("/" + EverEndpoints.stop)
 def stop(
-    request: Request, credentials: HTTPBasicCredentials = Depends(security)
+    request: Request, credentials: Annotated[HTTPBasicCredentials, Depends(security)]
 ) -> Response:
     _log(request)
     _check_user(credentials)
@@ -184,7 +186,7 @@ def stop(
 async def start_experiment(
     request: Request,
     background_tasks: BackgroundTasks,
-    credentials: HTTPBasicCredentials = Depends(security),
+    credentials: Annotated[HTTPBasicCredentials, Depends(security)],
 ) -> Response:
     _log(request)
     _check_user(credentials)
@@ -194,9 +196,6 @@ async def start_experiment(
         runner = ExperimentRunner(config)
         try:
             background_tasks.add_task(runner.run)
-            shared_data.status = ExperimentStatus(
-                status=ExperimentState.running, message="Experiment started"
-            )
             # Assume only one unique running experiment per everserver instance
             # Ideally, we should return the experiment ID in the response here
             shared_data.config_path = config.config_path
@@ -213,14 +212,14 @@ async def start_experiment(
                 status=ExperimentState.failed,
                 message=f"Could not start experiment: {e!s}",
             )
-            logging.getLogger(EVERSERVER).exception(e)
+            logging.getLogger(EXPERIMENT_SERVER).exception(e)
             return Response(f"Could not start experiment: {e!s}", status_code=501)
     return Response("Everest experiment is running")
 
 
 @router.get("/" + EverEndpoints.config_path)
 async def config_path(
-    request: Request, credentials: HTTPBasicCredentials = Depends(security)
+    request: Request, credentials: Annotated[HTTPBasicCredentials, Depends(security)]
 ) -> JSONResponse:
     _log(request)
     _check_user(credentials)
@@ -239,7 +238,7 @@ async def config_path(
 
 @router.get("/" + EverEndpoints.start_time)
 async def start_time(
-    request: Request, credentials: HTTPBasicCredentials = Depends(security)
+    request: Request, credentials: Annotated[HTTPBasicCredentials, Depends(security)]
 ) -> Response:
     _log(request)
     _check_user(credentials)
@@ -261,9 +260,9 @@ async def websocket_endpoint(websocket: WebSocket) -> None:
             if isinstance(event, EndEvent):
                 break
     except Exception as e:
-        logging.getLogger(EVERSERVER).exception(str(e))
+        logging.getLogger(EXPERIMENT_SERVER).exception(str(e))
     finally:
-        logging.getLogger(EVERSERVER).info(
+        logging.getLogger(EXPERIMENT_SERVER).info(
             f"Subscriber {subscriber_id} done. Closing websocket"
         )
         # Give some time for subscribers to get events
@@ -300,14 +299,16 @@ class ExperimentRunner:
 
     async def run(self) -> None:
         status_queue: SimpleQueue[StatusEvents] = SimpleQueue()
+        run_model: EverestRunModel | None = None
         try:
-            with ErtPluginContext() as runtime_plugins:
+            site_plugins = get_site_plugins()
+            with use_runtime_plugins(site_plugins):
                 run_model = EverestRunModel.create(
                     everest_config=self._everest_config,
                     experiment_name=f"EnOpt@{datetime.datetime.now().isoformat(timespec='seconds')}",
                     target_ensemble="batch",
                     status_queue=status_queue,
-                    runtime_plugins=runtime_plugins,
+                    runtime_plugins=site_plugins,
                 )
             shared_data.status = ExperimentStatus(
                 message="Experiment started", status=ExperimentState.running
@@ -316,9 +317,11 @@ class ExperimentRunner:
             simulation_future = loop.run_in_executor(
                 None,
                 lambda: run_model.start_simulations_thread(
-                    EvaluatorServerConfig()
-                    if run_model.queue_config.queue_system == QueueSystem.LOCAL
-                    else EvaluatorServerConfig(use_ipc_protocol=False)
+                    EvaluatorServerConfig(
+                        use_ipc_protocol=run_model.queue_config.queue_system
+                        == QueueSystem.LOCAL,
+                        prioritize_private_ip_address=site_plugins.prioritize_private_ip_address,
+                    )
                 ),
             )
             while True:
@@ -351,15 +354,18 @@ class ExperimentRunner:
                 status=exp_status,
             )
         except UserCancelled as e:
-            logging.getLogger(EVERSERVER).exception(e)
+            logging.getLogger(EXPERIMENT_SERVER).info(f"User cancelled: {e}")
         except Exception as e:
-            logging.getLogger(EVERSERVER).exception(e)
+            logging.getLogger(EXPERIMENT_SERVER).exception(e)
             shared_data.status = ExperimentStatus(
                 message=f"Exception: {e}\n{traceback.format_exc()}",
                 status=ExperimentState.failed,
             )
         finally:
-            logging.getLogger(EVERSERVER).info(
+            if run_model and run_model._experiment:
+                run_model._experiment.status = shared_data.status
+
+            logging.getLogger(EXPERIMENT_SERVER).info(
                 f"ExperimentRunner done. Items left in queue: {status_queue.qsize()}"
             )
 

ert/dark_storage/endpoints/experiments.py

@@ -29,7 +29,7 @@ def get_experiments(
             priors=create_priors(experiment),
             userdata={},
             parameters={
-                group: [m.model_dump() for m in config.metadata]
+                group: config.model_dump()
                 for group, config in experiment.parameter_configuration.items()
                 if not isinstance(config, SurfaceConfig)
             },
@@ -65,7 +65,7 @@ def get_experiment_by_id(
         priors=create_priors(experiment),
         userdata={},
         parameters={
-            group: [m.model_dump() for m in config.metadata]
+            group: config.model_dump()
             for group, config in experiment.parameter_configuration.items()
         },
         responses={

ert/dark_storage/endpoints/observations.py

@@ -1,7 +1,7 @@
 import json
 import logging
 import operator
-from typing import Any
+from typing import Annotated, Any
 from urllib.parse import unquote
 from uuid import UUID, uuid4
 
@@ -57,7 +57,9 @@ async def get_observations_for_response(
     storage: Storage = DEFAULT_STORAGE,
     ensemble_id: UUID,
     response_key: str,
-    filter_on: str | None = Query(None, description="JSON string with filters"),
+    filter_on: Annotated[
+        str | None, Query(description="JSON string with filters")
+    ] = None,
 ) -> list[js.ObservationOut]:
     response_key = unquote(response_key)
     try:
@@ -136,13 +138,13 @@ def _get_observations(
         df = df.with_columns(pl.Series(name="x_axis", values=df.map_rows(x_axis_fn)))
         df = df.sort("x_axis")
 
-        for obs_key, _obs_df in df.group_by("name"):
+        for obs_key, obs_df in df.group_by("name"):
             observations.append(
                 {
                     "name": obs_key[0],
-                    "values": _obs_df["values"].to_list(),
-                    "errors": _obs_df["errors"].to_list(),
-                    "x_axis": _obs_df["x_axis"].to_list(),
+                    "values": obs_df["values"].to_list(),
+                    "errors": obs_df["errors"].to_list(),
+                    "x_axis": obs_df["x_axis"].to_list(),
                 }
             )
 

ert/dark_storage/endpoints/parameters.py

@@ -113,20 +113,12 @@ def get_parameter_std_dev(
     return Response(content=buffer.getvalue(), media_type="application/octet-stream")
 
 
-def _extract_parameter_group_and_key(key: str) -> tuple[str, str] | tuple[None, None]:
-    key = key.removeprefix("LOG10_")
-    if ":" not in key:
-        # Assume all incoming keys are in format group:key for now
-        return None, None
-
-    param_group, param_key = key.split(":", maxsplit=1)
-    return param_group, param_key
-
-
 def data_for_parameter(ensemble: Ensemble, key: str) -> pd.DataFrame:
-    group, _ = _extract_parameter_group_and_key(key)
     try:
-        df = ensemble.load_scalars(group)
+        df = ensemble.load_scalar_keys([key], transformed=True)
+        if df.is_empty():
+            logger.warning(f"No data found for parameter '{key}'")
+            return pd.DataFrame()
     except KeyError as e:
         logger.error(e)
         return pd.DataFrame()

ert/dark_storage/endpoints/responses.py

@@ -45,7 +45,9 @@ async def get_response(
     storage: Storage = DEFAULT_STORAGE,
     ensemble_id: UUID,
     response_key: str,
-    filter_on: str | None = Query(None, description="JSON string with filters"),
+    filter_on: Annotated[
+        str | None, Query(description="JSON string with filters")
+    ] = None,
     accept: Annotated[str | None, Header()] = None,
 ) -> Response:
     try:
@@ -96,6 +98,7 @@ async def get_response(
 response_to_pandas_x_axis_fns: dict[str, Callable[[tuple[Any, ...]], Any]] = {
     "summary": lambda t: pd.Timestamp(t[2]).isoformat(),
     "gen_data": lambda t: str(t[3]),
+    "rft": lambda t: str(t[4]),
 }
 
 
@@ -116,7 +119,7 @@ def _extract_response_type_and_key(
 
 def data_for_response(
     ensemble: Ensemble, key: str, filter_on: dict[str, Any] | None = None
-) -> pd.DataFrame:
+) -> pd.DataFrame | pd.Series:
     response_key, response_type = _extract_response_type_and_key(
         key, ensemble.experiment.response_key_to_response_type
     )
@@ -147,10 +150,25 @@ def data_for_response(
         # This performs the same aggragation by mean of duplicate values
         # as in ert/analysis/_es_update.py
         df = df.groupby(["Date", "Realization"]).mean()
-        data = df.unstack(level="Date")
-        data.columns = data.columns.droplevel(0)
+        data = df.reset_index().pivot_table(
+            index="Realization", columns="Date", values=df.columns[0]
+        )
         return data.astype(float)
 
+    if response_type == "rft":
+        return (
+            ensemble.load_responses(
+                response_key,
+                tuple(realizations_with_responses),
+            )
+            .rename({"realization": "Realization"})
+            .select(["Realization", "depth", "values"])
+            .unique()
+            .to_pandas()
+            .pivot_table(index="Realization", columns="depth", values="values")
+            .reset_index(drop=True)
+        )
+
     if response_type == "gen_data":
         data = ensemble.load_responses(response_key, tuple(realizations_with_responses))
 
@@ -159,7 +177,7 @@ def data_for_response(
             assert "report_step" in filter_on
             report_step = int(filter_on["report_step"])
             vals = data.filter(pl.col("report_step").eq(report_step))
-            pivoted = vals.drop("response_key", "report_step").pivot(
+            pivoted = vals.drop("response_key", "report_step").pivot(  # noqa: PD010
                 on="index", values="values"
             )
             data = pivoted.to_pandas().set_index("realization")
@@ -169,3 +187,4 @@ def data_for_response(
 
         except (ValueError, KeyError, ColumnNotFoundError):
             return pd.DataFrame()
+    return pd.DataFrame()

ert/dark_storage/json_schema/ensemble.py

@@ -4,6 +4,8 @@ from uuid import UUID
 
 from pydantic import BaseModel
 
+from ert.storage.realization_storage_state import RealizationStorageState
+
 
 class _Ensemble(BaseModel):
     size: int
@@ -19,3 +21,4 @@ class EnsembleOut(_Ensemble):
     id: UUID
     experiment_id: UUID | None = None
     userdata: Mapping[str, Any]
+    realization_storage_states: Mapping[RealizationStorageState, int] | None = None

ert/dark_storage/json_schema/experiment.py

@@ -24,6 +24,6 @@ class ExperimentOut(_Experiment):
     ensemble_ids: list[UUID]
     priors: Mapping[str, dict[str, Any]]
     userdata: Mapping[str, Any]
-    parameters: Mapping[str, list[dict[str, Any]]]
+    parameters: Mapping[str, dict[str, Any]]
     responses: Mapping[str, list[dict[str, Any]]]
     observations: Mapping[str, dict[str, list[str]]]

ert/data/_measured_data.py

@@ -141,13 +141,14 @@ class MeasuredData:
 
         # Pandas differentiates vs int and str keys.
         # Legacy-wise we use int keys for realizations
-        pddf.rename(
-            columns={str(k): int(k) for k in active_realizations},
-            inplace=True,
+        pddf = (
+            pddf.rename(
+                columns={str(k): int(k) for k in active_realizations},
+            )
+            .set_index(["observation_key", "key_index"])
+            .transpose()
         )
 
-        pddf = pddf.set_index(["observation_key", "key_index"]).transpose()
-
         return pddf
 
 

ert/ensemble_evaluator/__init__.py

@@ -2,7 +2,13 @@ from ._ensemble import LegacyEnsemble as Ensemble
 from ._ensemble import Realization
 from .config import EvaluatorServerConfig
 from .evaluator import EnsembleEvaluator
-from .event import EndEvent, FullSnapshotEvent, SnapshotUpdateEvent, WarningEvent
+from .event import (
+    EndEvent,
+    FullSnapshotEvent,
+    SnapshotUpdateEvent,
+    StartEvent,
+    WarningEvent,
+)
 from .snapshot import EnsembleSnapshot, FMStepSnapshot, RealizationSnapshot
 
 __all__ = [
@@ -16,5 +22,6 @@ __all__ = [
     "Realization",
     "RealizationSnapshot",
     "SnapshotUpdateEvent",
+    "StartEvent",
     "WarningEvent",
 ]

ert/ensemble_evaluator/config.py

@@ -27,6 +27,7 @@ class EvaluatorServerConfig:
         use_token: bool = True,
         host: str | None = None,
         use_ipc_protocol: bool = True,
+        prioritize_private_ip_address: bool = False,
     ) -> None:
         self.host: str | None = host
         self.router_port: int | None = None
@@ -50,7 +51,7 @@ class EvaluatorServerConfig:
         if use_ipc_protocol:
             self.uri = f"ipc:///tmp/socket-{uuid.uuid4().hex[:8]}"
         elif self.host is None:
-            self.host = get_ip_address()
+            self.host = get_ip_address(prioritize_private_ip_address)
 
         if use_token:
             self.server_public_key, self.server_secret_key = zmq.curve_keypair()

ert/ensemble_evaluator/evaluator.py

@@ -6,6 +6,8 @@ import threading
 import traceback
 from collections import defaultdict
 from collections.abc import Awaitable, Callable, Iterable, Sequence
+from dataclasses import dataclass
+from math import ceil
 from typing import Any, cast, get_args
 
 import zmq.asyncio
@@ -15,6 +17,7 @@ from _ert.events import (
     EESnapshot,
     EESnapshotUpdate,
     EnsembleCancelled,
+    EnsembleEvaluationWarning,
     EnsembleFailed,
     EnsembleStarted,
     EnsembleSucceeded,
@@ -49,6 +52,13 @@ from .state import (
     ENSEMBLE_STATE_STOPPED,
 )
 
+
+@dataclass(order=True)
+class ParallelismViolation:
+    amount: float = 0
+    message: str = ""
+
+
 logger = logging.getLogger(__name__)
 
 EVENT_HANDLER = Callable[[list[SnapshotInputEvent]], Awaitable[None]]
@@ -68,6 +78,13 @@ class EventSentinel:
 
 class EnsembleEvaluator:
     BATCHING_INTERVAL = 0.5
+    DEFAULT_SLEEP_PERIOD = 0.1
+
+    # These properties help us determine whether the user
+    # has misconfigured NUM_CPU in their config.
+    ALLOWED_CPU_OVERSPENDING = 1.05
+    MINIMUM_WALLTIME_SECONDS = 30  # Information is only polled every 5 sec
+    CPU_OVERSPENDING_WARNING_THRESHOLD = 1.50
 
     def __init__(
         self,
@@ -123,6 +140,7 @@ class EnsembleEvaluator:
             submit_sleep=self.ensemble._queue_config.submit_sleep,
             ens_id=self.ensemble.id_,
         )
+        self.max_parallelism_violation = ParallelismViolation()
 
     async def _publisher(self) -> None:
         heartbeat_interval = 0.1
@@ -145,6 +163,11 @@ class EnsembleEvaluator:
                         self._evaluation_result.set_result(True)
                     return
 
+                elif isinstance(event, EnsembleEvaluationWarning):
+                    if self._event_handler:
+                        self._event_handler(event)
+                    self._events_to_send.task_done()
+
                 elif type(event) in {
                     EESnapshot,
                     EESnapshotUpdate,
@@ -191,7 +214,7 @@ class EnsembleEvaluator:
                 await self._signal_cancel()
                 logger.debug("Run model cancelled - during evaluation - cancel sent")
                 self._end_event.clear()
-            await asyncio.sleep(0.1)
+            await asyncio.sleep(self.DEFAULT_SLEEP_PERIOD)
 
     async def _send_terminate_message_to_dispatchers(self) -> None:
         event = TERMINATE_MSG
@@ -244,6 +267,7 @@ class EnsembleEvaluator:
                 event_handler[event_type] = func
 
         set_event_handler(set(get_args(FMEvent | RealizationEvent)), self._fm_handler)
+        set_event_handler({EnsembleEvaluationWarning}, self._warning_event_handler)
         set_event_handler({EnsembleStarted}, self._started_handler)
         set_event_handler({EnsembleSucceeded}, self._stopped_handler)
         set_event_handler({EnsembleCancelled}, self._cancelled_handler)
@@ -264,7 +288,7 @@ class EnsembleEvaluator:
                     batch.append((function, event))
                     self._events.task_done()
                 except asyncio.QueueEmpty:
-                    await asyncio.sleep(0.1)
+                    await asyncio.sleep(self.DEFAULT_SLEEP_PERIOD)
                     continue
             self._complete_batch.set()
             await self._batch_processing_queue.put(batch)
@@ -274,6 +298,12 @@ class EnsembleEvaluator:
     async def _fm_handler(self, events: Sequence[FMEvent | RealizationEvent]) -> None:
         await self._append_message(self.ensemble.update_snapshot(events))
 
+    async def _warning_event_handler(
+        self, events: Sequence[EnsembleEvaluationWarning]
+    ) -> None:
+        for event in events:
+            await self._events_to_send.put(event)
+
     async def _started_handler(self, events: Sequence[EnsembleStarted]) -> None:
         if self.ensemble.status != ENSEMBLE_STATE_FAILED:
             await self._append_message(self.ensemble.update_snapshot(events))
@@ -288,11 +318,9 @@ class EnsembleEvaluator:
             memory_usage = fm_step.get(ids.MAX_MEMORY_USAGE) or "-1"
             max_memory_usage = max(int(memory_usage), max_memory_usage)
 
-            cpu_message = detect_overspent_cpu(
+            self.detect_overspent_cpu(
                 self.ensemble.reals[int(real_id)].num_cpu, real_id, fm_step
             )
-            if self.ensemble.queue_system != QueueSystem.LOCAL and cpu_message:
-                logger.warning(cpu_message)
 
         logger.info(
             "Ensemble ran with maximum memory usage for a "
@@ -368,6 +396,7 @@ class EnsembleEvaluator:
                     logger.warning(
                         "Evaluator receiver closed, no new messages are received"
                     )
+                    return  # The socket is closed, and we won't re-establish it.
                 else:
                     logger.error(f"Unexpected error when listening to messages: {e}")
             except asyncio.CancelledError:
@@ -416,7 +445,7 @@ class EnsembleEvaluator:
             while True:
                 if self._evaluation_result.done():
                     break
-                await asyncio.sleep(0.1)
+                await asyncio.sleep(self.DEFAULT_SLEEP_PERIOD)
             logger.debug("Async server exiting.")
         finally:
             try:
@@ -647,27 +676,50 @@ class EnsembleEvaluator:
         else:
             await self._events.put(EnsembleCancelled(ensemble=self.ensemble.id_))
 
-
-def detect_overspent_cpu(num_cpu: int, real_id: str, fm_step: FMStepSnapshot) -> str:
-    """Produces a message warning about misconfiguration of NUM_CPU if
-    so is detected. Returns an empty string if everything is ok."""
-    allowed_overspending = 1.05
-    minimum_wallclock_time_seconds = 30  # Information is only polled every 5 sec
-
-    start_time = fm_step.get(ids.START_TIME)
-    end_time = fm_step.get(ids.END_TIME)
-    if start_time is None or end_time is None:
-        return ""
-    duration = (end_time - start_time).total_seconds()
-    if duration <= minimum_wallclock_time_seconds:
-        return ""
-    cpu_seconds = fm_step.get(ids.CPU_SECONDS) or 0.0
-    parallelization_obtained = cpu_seconds / duration
-    if parallelization_obtained > num_cpu * allowed_overspending:
-        return (
-            f"Misconfigured NUM_CPU, forward model step '{fm_step.get(ids.NAME)}' for "
-            f"realization {real_id} spent {cpu_seconds} cpu seconds "
-            f"with wall clock duration {duration:.1f} seconds, "
-            f"a factor of {parallelization_obtained:.2f}, while NUM_CPU was {num_cpu}."
+    def detect_overspent_cpu(
+        self, num_cpu: int, real_id: str, fm_step: FMStepSnapshot
+    ) -> None:
+        """Produces a message warning about misconfiguration of NUM_CPU if
+        so is detected. Returns an empty string if everything is ok."""
+        allowed_overspending = self.ALLOWED_CPU_OVERSPENDING * num_cpu
+        overspending_warning_threshold = (
+            self.CPU_OVERSPENDING_WARNING_THRESHOLD * num_cpu
         )
-    return ""
+
+        start_time = fm_step.get(ids.START_TIME)
+
+        end_time = fm_step.get(ids.END_TIME)
+        if start_time is None or end_time is None:
+            return
+
+        duration = (end_time - start_time).total_seconds()
+        if duration <= self.MINIMUM_WALLTIME_SECONDS:
+            return
+
+        cpu_seconds = fm_step.get(ids.CPU_SECONDS) or 0.0
+        parallelization_obtained = cpu_seconds / duration
+        if (
+            parallelization_obtained > allowed_overspending
+            and self.ensemble.queue_system != QueueSystem.LOCAL
+        ):
+            logger.warning(
+                f"Misconfigured NUM_CPU, forward model step '{fm_step.get(ids.NAME)}' "
+                f"for realization {real_id} spent {cpu_seconds} cpu seconds "
+                f"with wall clock duration {duration:.1f} seconds, a factor of "
+                f"{parallelization_obtained:.2f}, while NUM_CPU was {num_cpu}."
+            )
+            if parallelization_obtained > overspending_warning_threshold:
+                warning_msg = (
+                    "Overusage of CPUs detected!\n"
+                    f"Your experiment has used up to {ceil(parallelization_obtained)} "
+                    f"CPUs in step '{fm_step.get(ids.NAME)}', "
+                    f"while the Ert config has only requested {num_cpu}.\n"
+                    f"This means your experiment is consuming more CPU-resources than "
+                    f"requested and will slow down other users experiments.\n"
+                    f"We kindly ask you to set "
+                    f"NUM_CPU={ceil(parallelization_obtained)} in your Ert config."
+                )
+                self.max_parallelism_violation = max(
+                    self.max_parallelism_violation,
+                    ParallelismViolation(parallelization_obtained, warning_msg),
+                )

ert/ensemble_evaluator/event.py

@@ -1,4 +1,5 @@
 from collections.abc import Mapping
+from datetime import datetime
 from typing import Any, Literal
 
 from pydantic import BaseModel, ConfigDict, field_serializer, field_validator
@@ -42,6 +43,11 @@ class SnapshotUpdateEvent(_UpdateEvent):
     event_type: Literal["SnapshotUpdateEvent"] = "SnapshotUpdateEvent"
 
 
+class StartEvent(BaseModel):
+    event_type: Literal["StartEvent"] = "StartEvent"
+    timestamp: datetime
+
+
 class EndEvent(BaseModel):
     model_config = ConfigDict(arbitrary_types_allowed=True, extra="forbid")
     event_type: Literal["EndEvent"] = "EndEvent"

ert/ensemble_evaluator/snapshot.py

@@ -12,6 +12,7 @@ from _ert.events import (
     EESnapshot,
     EESnapshotUpdate,
     EnsembleCancelled,
+    EnsembleEvaluationWarning,
     EnsembleEvent,
     EnsembleFailed,
     EnsembleStarted,
@@ -424,7 +425,8 @@ class EnsembleSnapshot:
 
         elif e_type in get_args(EnsembleEvent):
             event = cast(EnsembleEvent, event)
-            self._ensemble_state = _ENSEMBLE_TYPE_EVENT_TO_STATUS[type(event)]
+            if not isinstance(event, EnsembleEvaluationWarning):
+                self._ensemble_state = _ENSEMBLE_TYPE_EVENT_TO_STATUS[type(event)]
         elif type(event) is EESnapshotUpdate:
             self.merge_snapshot(EnsembleSnapshot.from_nested_dict(event.snapshot))
         elif type(event) is EESnapshot:

ert/ensemble_evaluator/state.py

@@ -7,6 +7,7 @@ COLOR_RUNNING: Final = (255, 255, 153)
 COLOR_UNKNOWN: Final = (128, 128, 128)
 COLOR_WAITING: Final = (164, 200, 255)
 COLOR_CANCELLED: Final = (235, 242, 246)
+COLOR_WARNING: Final = (255, 103, 0)
 
 ENSEMBLE_STATE_CANCELLED: Final = "Cancelled"
 ENSEMBLE_STATE_FAILED: Final = "Failed"