ert 17.0.0__py3-none-any.whl → 19.0.0rc2__py3-none-any.whl

_ert/events.py

@@ -41,10 +41,12 @@ class Id:
     ENSEMBLE_SUCCEEDED_TYPE = Literal["ensemble.succeeded"]
     ENSEMBLE_CANCELLED_TYPE = Literal["ensemble.cancelled"]
     ENSEMBLE_FAILED_TYPE = Literal["ensemble.failed"]
+    ENSEMBLE_WARNING_TYPE = Literal["ensemble.warning"]
     ENSEMBLE_STARTED: Final = "ensemble.started"
     ENSEMBLE_SUCCEEDED: Final = "ensemble.succeeded"
     ENSEMBLE_CANCELLED: Final = "ensemble.cancelled"
     ENSEMBLE_FAILED: Final = "ensemble.failed"
+    ENSEMBLE_WARNING: Final = "ensemble.warning"
     ENSEMBLE_TYPES = (
         ENSEMBLE_STARTED_TYPE
         | ENSEMBLE_FAILED_TYPE
@@ -170,6 +172,15 @@ class EnsembleCancelled(EnsembleBaseEvent):
     event_type: Id.ENSEMBLE_CANCELLED_TYPE = Id.ENSEMBLE_CANCELLED
 
 
+class EnsembleEvaluationWarning(EnsembleBaseEvent):
+    """This event is to indicate that something unexpected happened while
+    running the ensemble, and that it might be stuck in an unresponsive state.
+    """
+
+    event_type: Id.ENSEMBLE_WARNING_TYPE = Id.ENSEMBLE_WARNING
+    warning_message: str
+
+
 class EESnapshot(EnsembleBaseEvent):
     event_type: Id.EE_SNAPSHOT_TYPE = Id.EE_SNAPSHOT
     snapshot: Any
@@ -200,9 +211,15 @@ RealizationEvent = (
     | RealizationResubmit
 )
 
-EnsembleEvent = EnsembleStarted | EnsembleSucceeded | EnsembleFailed | EnsembleCancelled
+EnsembleEvent = (
+    EnsembleStarted
+    | EnsembleSucceeded
+    | EnsembleFailed
+    | EnsembleCancelled
+    | EnsembleEvaluationWarning
+)
 
-EEEvent = EESnapshot | EESnapshotUpdate
+EEEvent = EESnapshot | EESnapshotUpdate | EnsembleEvaluationWarning
 
 SnapshotInputEvent = RealizationEvent | EnsembleEvent | FMEvent
 

_ert/forward_model_runner/client.py

@@ -2,8 +2,9 @@ from __future__ import annotations
 
 import asyncio
 import logging
+import types
 import uuid
-from typing import Any, Self
+from typing import Self
 
 import zmq
 import zmq.asyncio
@@ -67,7 +68,10 @@ class Client:
         return self
 
     async def __aexit__(
-        self, exc_type: Any, exc_value: Any, exc_traceback: Any
+        self,
+        exc_type: type[BaseException] | None,
+        exc_value: BaseException | None,
+        exc_traceback: types.TracebackType | None,
     ) -> None:
         try:
             await self.send(DISCONNECT_MSG)

ert/__main__.py

@@ -35,8 +35,9 @@ from ert.mode_definitions import (
 )
 from ert.namespace import Namespace
 from ert.plugins import ErtRuntimePlugins, get_site_plugins, setup_site_logging
-from ert.run_models.multiple_data_assimilation import MultipleDataAssimilation
-from ert.services import StorageService, WebvizErt
+from ert.run_models.multiple_data_assimilation import MultipleDataAssimilationConfig
+from ert.services import ErtServer, WebvizErt
+from ert.shared.status.utils import get_ert_memory_usage
 from ert.shared.storage.command import add_parser_options as ert_api_add_parser_options
 from ert.storage import ErtStorageException, ErtStoragePermissionError
 from ert.trace import trace, tracer
@@ -53,9 +54,9 @@ logger = logging.getLogger(__name__)
 
 
 def run_ert_storage(args: Namespace, _: ErtRuntimePlugins | None = None) -> None:
-    with StorageService.start_server(
+    with ErtServer.start_server(
         verbose=True,
-        project=ErtConfig.from_file(args.config).ens_path,
+        project=Path(ErtConfig.from_file(args.config).ens_path),
         parent_pid=os.getpid(),
     ) as server:
         server.wait()
@@ -79,18 +80,32 @@ def run_webviz_ert(args: Namespace, _: ErtRuntimePlugins | None = None) -> None:
     # only use the base name of the config file path
     kwargs["ert_config"] = os.path.basename(args.config)
     kwargs["project"] = os.path.abspath(ens_path)
+
+    yellow = "\x1b[33m"
+    green = "\x1b[32m"
+    bold = "\x1b[1m"
+    reset = "\x1b[0m"
+
     try:
-        with StorageService.init_service(project=os.path.abspath(ens_path)) as storage:
+        with ErtServer.init_service(project=Path(ens_path).absolute()) as storage:
             storage.wait_until_ready()
             print(
-                """
-    -----------------------------------------------------------
+                f"""
+    ---------------------------------------------------------------
+
+    {yellow}{bold}Webviz-ERT is deprecated and will be removed in the near future{reset}
+
+    {green}{bold}Plotting capabilities provided by Webviz-ERT are now available
+    using the ERT plotter{reset}
+
+    ---------------------------------------------------------------
 
     Starting up Webviz-ERT. This might take more than a minute.
 
-    -----------------------------------------------------------
+    ---------------------------------------------------------------
     """
             )
+            logger.info("Show Webviz-ert deprecation warning")
             webviz_kwargs = {
                 "experimental_mode": args.experimental_mode,
                 "verbose": args.verbose,
@@ -501,7 +516,7 @@ def get_ert_parser(parser: ArgumentParser | None = None) -> ArgumentParser:
     es_mda_parser.add_argument(
         "--weights",
         type=valid_weights,
-        default=MultipleDataAssimilation.default_weights,
+        default=MultipleDataAssimilationConfig.default_weights,
         help="Example custom relative weights: '8,4,2,1'. This means multiple data "
         "assimilation ensemble smoother will half the weight applied to the "
         "observation errors from one iteration to the next across 4 iterations.",
@@ -566,15 +581,15 @@ def get_ert_parser(parser: ArgumentParser | None = None) -> ArgumentParser:
             "--color-always",
             action="store_true",
             help="Force coloring of monitor output, which is automatically"
-            + " disabled if the output stream is not a terminal.",
+            " disabled if the output stream is not a terminal.",
             default=False,
         )
         cli_parser.add_argument(
             "--disable-monitoring",
             action="store_true",
             help="Monitoring will continuously print the status of the realisations"
-            + " classified into Waiting, Pending, Running, Failed, Finished"
-            + " and Unknown.",
+            " classified into Waiting, Pending, Running, Failed, Finished"
+            " and Unknown.",
             default=False,
         )
         cli_parser.add_argument(
@@ -598,7 +613,7 @@ def ert_parser(parser: ArgumentParser | None, args: Sequence[str]) -> Namespace:
 def log_process_usage() -> None:
     try:
         usage = resource.getrusage(resource.RUSAGE_SELF)
-        max_rss = ert.shared.status.utils.get_ert_memory_usage()
+        max_rss = get_ert_memory_usage()
 
         usage_dict: dict[str, int | float] = {
             "User time": usage.ru_utime,

ert/analysis/_enif_update.py

@@ -160,10 +160,14 @@ def analysis_EnIF(
         Y=S.T,
         verbose_level=5,
     )
-
+    updated_parameters = [
+        p
+        for p, config in source_ensemble.experiment.parameter_configuration.items()
+        if config.update
+    ]
     # Learn the precision matrix block-sparse over parameter groups
     Prec_u = sp.sparse.csc_matrix((0, 0), dtype=float)
-    for param_group in parameters:
+    for param_group in updated_parameters:
         config_node = source_ensemble.experiment.parameter_configuration[param_group]
         X_local = source_ensemble.load_parameters_numpy(param_group, iens_active_index)
         X_local_scaler = StandardScaler()
@@ -215,7 +219,7 @@ def analysis_EnIF(
 
     # Iterate over parameters to store the updated ensemble
     parameters_updated = 0
-    for param_group in parameters:
+    for param_group in updated_parameters:
         log_msg = f"Storing data for {param_group}.."
         logger.info(log_msg)
         progress_callback(AnalysisStatusEvent(msg=log_msg))
@@ -241,7 +245,7 @@ def analysis_EnIF(
         )
     _copy_unupdated_parameters(
         list(source_ensemble.experiment.parameter_configuration.keys()),
-        parameters,
+        updated_parameters,
         iens_active_index,
         source_ensemble,
         target_ensemble,

ert/analysis/_es_update.py

@@ -2,6 +2,7 @@ from __future__ import annotations
 
 import functools
 import logging
+import re
 import time
 import warnings
 from collections.abc import Callable, Iterable, Sequence
@@ -441,6 +442,12 @@ def smoother_update(
         with warnings.catch_warnings():
             original_showwarning = warnings.showwarning
 
+            ILL_CONDITIONED_RE = re.compile(
+                r"^LinAlgWarning:.*ill[- ]?conditioned\s+matrix", re.IGNORECASE
+            )
+            LIMIT_ILL_CONDITIONED_WARNING = 1000
+            illconditioned_warn_counter = 0
+
             def log_warning(
                 message: Warning | str,
                 category: type[Warning],
@@ -449,12 +456,18 @@ def smoother_update(
                 file: TextIO | None = None,
                 line: str | None = None,
             ) -> None:
-                logger.warning(
-                    f"{category.__name__}: {message} (from {filename}:{lineno})"
-                )
-                original_showwarning(
-                    message, category, filename, lineno, file=file, line=line
-                )
+                nonlocal illconditioned_warn_counter
+
+                if ILL_CONDITIONED_RE.search(str(message)):
+                    illconditioned_warn_counter += 1
+
+                if illconditioned_warn_counter < LIMIT_ILL_CONDITIONED_WARNING:
+                    logger.warning(
+                        f"{category.__name__}: {message} (from {filename}:{lineno})"
+                    )
+                    original_showwarning(
+                        message, category, filename, lineno, file=file, line=line
+                    )
 
             warnings.showwarning = log_warning
             analysis_ES(

ert/analysis/_update_commons.py

@@ -61,10 +61,19 @@ def _copy_unupdated_parameters(
 
     # Copy the non-updated parameter groups from source to target
     # for each active realization
+    complete_df: pl.DataFrame | None = None
     for parameter_group in not_updated_parameter_groups:
-        source_ensemble.experiment.parameter_configuration[
-            parameter_group
-        ].copy_parameters(source_ensemble, target_ensemble, iens_active_index)
+        data = source_ensemble.load_parameters(parameter_group, iens_active_index)
+        if isinstance(data, pl.DataFrame):
+            if complete_df is None:
+                complete_df = data
+            else:
+                complete_df = complete_df.join(data, on="realization")
+        else:
+            target_ensemble.save_parameters(dataset=data)
+
+    if complete_df is not None:
+        target_ensemble.save_parameters(complete_df)
 
 
 def _expand_wildcards(
@@ -331,11 +340,12 @@ def _all_parameters(
 ) -> npt.NDArray[np.float64]:
     """Return all parameters in assimilation problem"""
 
-    param_groups = list(ensemble.experiment.parameter_configuration.keys())
-
+    groups_to_update = [
+        k for k, v in ensemble.experiment.parameter_configuration.items() if v.update
+    ]
     param_arrays = [
         ensemble.load_parameters_numpy(param_group, iens_active_index)
-        for param_group in param_groups
+        for param_group in groups_to_update
     ]
 
     return np.vstack(param_arrays)

ert/cli/main.py

@@ -1,4 +1,3 @@
-#!/usr/bin/env python
 from __future__ import annotations
 
 import contextlib
@@ -7,6 +6,7 @@ import os
 import queue
 import sys
 from collections import Counter
+from pathlib import Path
 from typing import TextIO
 
 from _ert.threading import ErtThread
@@ -27,7 +27,7 @@ from ert.namespace import Namespace
 from ert.plugins import ErtRuntimePlugins, get_site_plugins
 from ert.run_models.event import StatusEvents
 from ert.run_models.model_factory import create_model
-from ert.storage import open_storage
+from ert.storage import LocalStorage, open_storage
 from ert.storage.local_storage import local_storage_set_ert_config
 
 
@@ -88,11 +88,15 @@ def run_cli(args: Namespace, runtime_plugins: ErtRuntimePlugins | None = None) -
             )
 
     if args.mode == WORKFLOW_MODE:
-        with open_storage(ert_config.ens_path, "w") as storage:
+        path = Path(ert_config.ens_path)
+        if LocalStorage.check_migration_needed(path):
+            LocalStorage.perform_migration(path)
+        with open_storage(path, "w") as storage:
             execute_workflow(ert_config, storage, args.name)
         return
 
     status_queue: queue.SimpleQueue[StatusEvents] = queue.SimpleQueue()
+    using_local_queuesystem: bool = True
     try:
         with use_runtime_plugins(get_site_plugins()):
             model = create_model(
@@ -100,6 +104,9 @@ def run_cli(args: Namespace, runtime_plugins: ErtRuntimePlugins | None = None) -
                 args,
                 status_queue,
             )
+            using_local_queuesystem = (
+                ert_config.queue_config.queue_system == QueueSystem.LOCAL
+            )
     except ValueError as e:
         raise ErtCliError(f"{args.mode} was not valid, failed with: {e}") from e
 
@@ -116,17 +123,17 @@ def run_cli(args: Namespace, runtime_plugins: ErtRuntimePlugins | None = None) -
             f"and DESIGN_MATRIX ({model.active_realizations.count(True)})"
         )
 
-    if args.port_range is None and model.queue_system == QueueSystem.LOCAL:
+    if args.port_range is None and using_local_queuesystem:
         # This is within the range for ephemeral ports as defined by
         # most unix flavors https://en.wikipedia.org/wiki/Ephemeral_port
         args.port_range = range(49152, 51819)
 
-    use_ipc_protocol = model.queue_system == QueueSystem.LOCAL
     evaluator_server_config = EvaluatorServerConfig(
         port_range=None
         if args.port_range is None
         else (min(args.port_range), max(args.port_range) + 1),
-        use_ipc_protocol=use_ipc_protocol,
+        use_ipc_protocol=using_local_queuesystem,
+        prioritize_private_ip_address=ert_config.prioritize_private_ip_address,
     )
 
     if model.check_if_runpath_exists():

ert/cli/monitor.py

@@ -9,6 +9,7 @@ from typing import TextIO
 import humanize
 from tqdm import tqdm
 
+from _ert.events import EnsembleEvaluationWarning
 from ert.ensemble_evaluator import (
     EndEvent,
     EnsembleSnapshot,
@@ -19,6 +20,7 @@ from ert.ensemble_evaluator import identifiers as ids
 from ert.ensemble_evaluator.state import (
     COLOR_FAILED,
     COLOR_FINISHED,
+    COLOR_WARNING,
     FORWARD_MODEL_STATE_FAILURE,
     REAL_STATE_TO_COLOR,
 )
@@ -96,6 +98,11 @@ class Monitor:
                     | RunModelErrorEvent() as event
                 ):
                     event.write_as_csv(output_path)
+                case EnsembleEvaluationWarning(warning_message=msg):
+                    print(
+                        self._colorize(msg, color=COLOR_WARNING),
+                        file=self._out,
+                    )
 
     def _print_step_errors(self) -> None:
         failed_steps: dict[str | None, int] = {}

ert/config/__init__.py

@@ -11,9 +11,8 @@ from .ensemble_config import EnsembleConfig
 from .ert_config import ErtConfig, forward_model_step_from_config_contents
 from .ert_plugin import ErtPlugin
 from .ert_script import ErtScript
-from .everest_constraints_config import EverestConstraintsConfig
-from .everest_objective_config import EverestObjectivesConfig
-from .ext_param_config import ExtParamConfig
+from .everest_control import EverestControl, SamplerConfig
+from .everest_response import EverestConstraintsConfig, EverestObjectivesConfig
 from .external_ert_script import ExternalErtScript
 from .field import Field, field_transform
 from .forward_model_step import (
@@ -23,12 +22,16 @@ from .forward_model_step import (
     ForwardModelStepPlugin,
     ForwardModelStepValidationError,
     ForwardModelStepWarning,
+    SiteInstalledForwardModelStep,
+    SiteOrUserForwardModelStep,
+    UserInstalledForwardModelStep,
 )
 from .gen_data_config import GenDataConfig
 from .gen_kw_config import DataSource, GenKwConfig, PriorDict
+from .known_response_types import KnownResponseTypes
 from .lint_file import lint_file
 from .model_config import ModelConfig
-from .parameter_config import ParameterCardinality, ParameterConfig, ParameterMetadata
+from .parameter_config import ParameterCardinality, ParameterConfig
 from .parsing import (
     ConfigValidationError,
     ConfigWarning,
@@ -45,6 +48,7 @@ from .queue_config import (
     QueueConfig,
 )
 from .response_config import InvalidResponseFile, ResponseConfig, ResponseMetadata
+from .rft_config import RFTConfig
 from .summary_config import SummaryConfig
 from .surface_config import SurfaceConfig
 from .workflow import Workflow
@@ -86,9 +90,9 @@ __all__ = [
     "ErtScript",
     "ErtScriptWorkflow",
     "EverestConstraintsConfig",
+    "EverestControl",
     "EverestObjectivesConfig",
     "ExecutableWorkflow",
-    "ExtParamConfig",
     "ExternalErtScript",
     "Field",
     "ForwardModelStep",
@@ -105,6 +109,7 @@ __all__ = [
     "InversionTypeES",
     "KnownQueueOptions",
     "KnownQueueOptionsAdapter",
+    "KnownResponseTypes",
     "LegacyWorkflowConfigs",
     "LocalQueueOptions",
     "ModelConfig",
@@ -114,7 +119,6 @@ __all__ = [
     "OutlierSettings",
     "ParameterCardinality",
     "ParameterConfig",
-    "ParameterMetadata",
     "PostExperimentFixtures",
     "PostSimulationFixtures",
     "PostUpdateFixtures",
@@ -125,10 +129,15 @@ __all__ = [
     "PriorDict",
     "QueueConfig",
     "QueueSystem",
+    "RFTConfig",
     "ResponseConfig",
     "ResponseMetadata",
+    "SamplerConfig",
+    "SiteInstalledForwardModelStep",
+    "SiteOrUserForwardModelStep",
     "SummaryConfig",
     "SurfaceConfig",
+    "UserInstalledForwardModelStep",
     "WarningInfo",
     "Workflow",
     "WorkflowConfigs",